From d13c9206061e9f95a4438e60148b5f6bb24c20d5 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Mon, 12 Jul 2021 12:34:05 -0700
Subject: [PATCH 01/79] Svc: Move ETP clearing to dropTypeDependentFields

I  am not sure why ExternalTrafficPolicy was different, but this is more
consistent with other field clearing logic.
---
 pkg/registry/core/service/storage/rest.go | 23 +----------------------
 pkg/registry/core/service/strategy.go     | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 5c499be9a97..0b30eb7cf07 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -346,25 +346,6 @@ func shouldAllocateNodePorts(service *api.Service) bool {
 	return false
 }
 
-// externalTrafficPolicyUpdate adjusts ExternalTrafficPolicy during service update if needed.
-// It is necessary because we default ExternalTrafficPolicy field to different values.
-// (NodePort / LoadBalancer: default is Global; Other types: default is empty.)
-func externalTrafficPolicyUpdate(oldService, service *api.Service) {
-	var neededExternalTraffic, needsExternalTraffic bool
-	if oldService.Spec.Type == api.ServiceTypeNodePort ||
-		oldService.Spec.Type == api.ServiceTypeLoadBalancer {
-		neededExternalTraffic = true
-	}
-	if service.Spec.Type == api.ServiceTypeNodePort ||
-		service.Spec.Type == api.ServiceTypeLoadBalancer {
-		needsExternalTraffic = true
-	}
-	if neededExternalTraffic && !needsExternalTraffic {
-		// Clear ExternalTrafficPolicy to prevent confusion from ineffective field.
-		service.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyType("")
-	}
-}
-
 // healthCheckNodePortUpdate handles HealthCheckNodePort allocation/release
 // and adjusts HealthCheckNodePort during service update if needed.
 func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
@@ -390,13 +371,12 @@ func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service, node
 		klog.Infof("Transition to non LoadBalancer type service or LoadBalancer type service with ExternalTrafficPolicy=Global")
 		klog.V(4).Infof("Releasing healthCheckNodePort: %d", oldHealthCheckNodePort)
 		nodePortOp.ReleaseDeferred(int(oldHealthCheckNodePort))
-		// Clear the HealthCheckNodePort field.
-		service.Spec.HealthCheckNodePort = 0
 
 	// Case 3: Remain in needs HealthCheckNodePort.
 	// Reject changing the value of the HealthCheckNodePort field.
 	case neededHealthCheckNodePort && needsHealthCheckNodePort:
 		if oldHealthCheckNodePort != newHealthCheckNodePort {
+			//FIXME: Let validation do this.
 			klog.Warningf("Attempt to change value of health check node port DENIED")
 			fldPath := field.NewPath("spec", "healthCheckNodePort")
 			el := field.ErrorList{field.Invalid(fldPath, newHealthCheckNodePort,
@@ -499,7 +479,6 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	if !success || err != nil {
 		return nil, false, err
 	}
-	externalTrafficPolicyUpdate(oldService, service)
 	if errs := validation.ValidateServiceExternalTrafficFieldsCombination(service); len(errs) > 0 {
 		return nil, false, errors.NewInvalid(api.Kind("Service"), service.Name, errs)
 	}
diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go
index 53fe07cdbec..10f1f985339 100644
--- a/pkg/registry/core/service/strategy.go
+++ b/pkg/registry/core/service/strategy.go
@@ -509,6 +509,12 @@ func dropTypeDependentFields(newSvc *api.Service, oldSvc *api.Service) {
 		newSvc.Spec.LoadBalancerClass = nil
 	}
 
+	// If a user is switching to a type that doesn't need ExternalTrafficPolicy
+	// AND they did not change this field, it is safe to drop it.
+	if needsExternalTrafficPolicy(oldSvc) && !needsExternalTrafficPolicy(newSvc) && sameExternalTrafficPolicy(oldSvc, newSvc) {
+		newSvc.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyType("")
+	}
+
 	// NOTE: there are other fields like `selector` which we could wipe.
 	// Historically we did not wipe them and they are not allocated from
 	// finite pools, so we are (currently) choosing to leave them alone.
@@ -597,6 +603,14 @@ func sameLoadBalancerClass(oldSvc, newSvc *api.Service) bool {
 	return *oldSvc.Spec.LoadBalancerClass == *newSvc.Spec.LoadBalancerClass
 }
 
+func needsExternalTrafficPolicy(svc *api.Service) bool {
+	return svc.Spec.Type == api.ServiceTypeNodePort || svc.Spec.Type == api.ServiceTypeLoadBalancer
+}
+
+func sameExternalTrafficPolicy(oldSvc, newSvc *api.Service) bool {
+	return oldSvc.Spec.ExternalTrafficPolicy == newSvc.Spec.ExternalTrafficPolicy
+}
+
 // this func allows user to downgrade a service by just changing
 // IPFamilyPolicy to SingleStack
 func trimFieldsForDualStackDowngrade(newService, oldService *api.Service) {

From 89587b3c6a692905a6534efbab4cf03f8b9a4c5d Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 17 Nov 2020 20:42:16 -0800
Subject: [PATCH 02/79] Svc REST: Encapsulate IP and Port allocator logic

Encapsulate the allocator logic so it can be shared across REST
layers while we stage a series of commits to get rid of one layer.
---
 pkg/registry/core/rest/storage_core.go        |  14 ++-
 pkg/registry/core/service/storage/rest.go     | 107 ++++++++----------
 .../core/service/storage/rest_test.go         |  94 ++++++++-------
 3 files changed, 109 insertions(+), 106 deletions(-)

diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go
index 05e6ae1bcd3..9f5d7a668c5 100644
--- a/pkg/registry/core/rest/storage_core.go
+++ b/pkg/registry/core/rest/storage_core.go
@@ -254,16 +254,24 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err
 	}
 
+	serviceIPAllocators := map[api.IPFamily]ipallocator.Interface{
+		serviceClusterIPAllocator.IPFamily(): serviceClusterIPAllocator,
+	}
+	if secondaryServiceClusterIPAllocator != nil {
+		serviceIPAllocators[secondaryServiceClusterIPAllocator.IPFamily()] = secondaryServiceClusterIPAllocator
+	}
+
 	serviceRESTStorage, serviceStatusStorage, err := servicestore.NewGenericREST(restOptionsGetter, serviceClusterIPRange, secondaryServiceClusterIPAllocator != nil)
 	if err != nil {
 		return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err
 	}
 
-	serviceRest, serviceRestProxy := servicestore.NewREST(serviceRESTStorage,
+	serviceRest, serviceRestProxy := servicestore.NewREST(
+		serviceRESTStorage,
 		endpointsStorage,
 		podStorage.Pod,
-		serviceClusterIPAllocator,
-		secondaryServiceClusterIPAllocator,
+		serviceClusterIPAllocator.IPFamily(),
+		serviceIPAllocators,
 		serviceNodePortAllocator,
 		c.ProxyTransport)
 
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 0b30eb7cf07..1af49f057e2 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -51,14 +51,20 @@ import (
 
 // REST adapts a service registry into apiserver's RESTStorage model.
 type REST struct {
-	strategy                    rest.RESTCreateUpdateStrategy
-	services                    ServiceStorage
-	endpoints                   EndpointsStorage
+	strategy       rest.RESTCreateUpdateStrategy
+	services       ServiceStorage
+	endpoints      EndpointsStorage
+	alloc          RESTAllocStuff
+	proxyTransport http.RoundTripper
+	pods           rest.Getter
+}
+
+// RESTAllocStuff is a temporary struct to facilitate the flattening of service
+// REST layers.  It will be cleaned up over a series of commits.
+type RESTAllocStuff struct {
 	serviceIPAllocatorsByFamily map[api.IPFamily]ipallocator.Interface
 	defaultServiceIPFamily      api.IPFamily // --service-cluster-ip-range[0]
 	serviceNodePorts            portallocator.Interface
-	proxyTransport              http.RoundTripper
-	pods                        rest.Getter
 }
 
 // ServiceNodePort includes protocol and port number of a service NodePort.
@@ -95,54 +101,37 @@ func NewREST(
 	services ServiceStorage,
 	endpoints EndpointsStorage,
 	pods rest.Getter,
-	serviceIPs ipallocator.Interface,
-	secondaryServiceIPs ipallocator.Interface,
-	serviceNodePorts portallocator.Interface,
+	defaultFamily api.IPFamily,
+	ipAllocs map[api.IPFamily]ipallocator.Interface,
+	portAlloc portallocator.Interface,
 	proxyTransport http.RoundTripper,
 ) (*REST, *registry.ProxyREST) {
 
-	strategy, _ := registry.StrategyForServiceCIDRs(serviceIPs.CIDR(), secondaryServiceIPs != nil)
+	strategy, _ := registry.StrategyForServiceCIDRs(ipAllocs[defaultFamily].CIDR(), len(ipAllocs) > 1)
 
-	byIPFamily := make(map[api.IPFamily]ipallocator.Interface)
-
-	// detect this cluster default Service IPFamily (ipfamily of --service-cluster-ip-range[0])
-	serviceIPFamily := api.IPv4Protocol
-	cidr := serviceIPs.CIDR()
-	if netutils.IsIPv6CIDR(&cidr) {
-		serviceIPFamily = api.IPv6Protocol
-	}
-
-	// add primary family
-	byIPFamily[serviceIPFamily] = serviceIPs
-
-	if secondaryServiceIPs != nil {
-		// process secondary family
-		secondaryServiceIPFamily := api.IPv6Protocol
-
-		// get family of secondary
-		if serviceIPFamily == api.IPv6Protocol {
-			secondaryServiceIPFamily = api.IPv4Protocol
-		}
-		// add it
-		byIPFamily[secondaryServiceIPFamily] = secondaryServiceIPs
-	}
-
-	klog.V(0).Infof("the default service ipfamily for this cluster is: %s", string(serviceIPFamily))
+	klog.V(0).Infof("the default service ipfamily for this cluster is: %s", string(defaultFamily))
 
 	rest := &REST{
-		strategy:                    strategy,
-		services:                    services,
-		endpoints:                   endpoints,
-		serviceIPAllocatorsByFamily: byIPFamily,
-		serviceNodePorts:            serviceNodePorts,
-		defaultServiceIPFamily:      serviceIPFamily,
-		proxyTransport:              proxyTransport,
-		pods:                        pods,
+		strategy:       strategy,
+		services:       services,
+		endpoints:      endpoints,
+		proxyTransport: proxyTransport,
+		pods:           pods,
+		alloc:          makeAlloc(defaultFamily, ipAllocs, portAlloc),
 	}
 
 	return rest, &registry.ProxyREST{Redirector: rest, ProxyTransport: proxyTransport}
 }
 
+// This is a trasitionary function to facilitate service REST flattening.
+func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) RESTAllocStuff {
+	return RESTAllocStuff{
+		defaultServiceIPFamily:      defaultFamily,
+		serviceIPAllocatorsByFamily: ipAllocs,
+		serviceNodePorts:            portAlloc,
+	}
+}
+
 var (
 	_ ServiceStorage              = &REST{}
 	_ rest.CategoriesProvider     = &REST{}
@@ -226,7 +215,7 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 		}
 	}
 
-	nodePortOp := portallocator.StartOperation(rs.serviceNodePorts, dryrun.IsDryRun(options.DryRun))
+	nodePortOp := portallocator.StartOperation(rs.alloc.serviceNodePorts, dryrun.IsDryRun(options.DryRun))
 	defer nodePortOp.Finish()
 
 	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
@@ -314,7 +303,7 @@ func (rs *REST) releaseAllocatedResources(svc *api.Service) {
 	rs.releaseServiceClusterIPs(svc)
 
 	for _, nodePort := range collectServiceNodePorts(svc) {
-		err := rs.serviceNodePorts.Release(nodePort)
+		err := rs.alloc.serviceNodePorts.Release(nodePort)
 		if err != nil {
 			// these should be caught by an eventual reconciliation / restart
 			utilruntime.HandleError(fmt.Errorf("Error releasing service %s node port %d: %v", svc.Name, nodePort, err))
@@ -324,7 +313,7 @@ func (rs *REST) releaseAllocatedResources(svc *api.Service) {
 	if apiservice.NeedsHealthCheck(svc) {
 		nodePort := svc.Spec.HealthCheckNodePort
 		if nodePort > 0 {
-			err := rs.serviceNodePorts.Release(int(nodePort))
+			err := rs.alloc.serviceNodePorts.Release(int(nodePort))
 			if err != nil {
 				// these should be caught by an eventual reconciliation / restart
 				utilruntime.HandleError(fmt.Errorf("Error releasing service %s health check node port %d: %v", svc.Name, nodePort, err))
@@ -443,7 +432,7 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 		}
 	}()
 
-	nodePortOp := portallocator.StartOperation(rs.serviceNodePorts, dryrun.IsDryRun(options.DryRun))
+	nodePortOp := portallocator.StartOperation(rs.alloc.serviceNodePorts, dryrun.IsDryRun(options.DryRun))
 	defer nodePortOp.Finish()
 
 	// try set ip families (for missing ip families)
@@ -586,7 +575,7 @@ func (rs *REST) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]s
 	allocated := make(map[api.IPFamily]string)
 
 	for family, ip := range toAlloc {
-		allocator := rs.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
+		allocator := rs.alloc.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
 		if ip == "" {
 			allocatedIP, err := allocator.AllocateNext()
 			if err != nil {
@@ -613,7 +602,7 @@ func (rs *REST) releaseClusterIPs(toRelease map[api.IPFamily]string) (map[api.IP
 
 	released := make(map[api.IPFamily]string)
 	for family, ip := range toRelease {
-		allocator, ok := rs.serviceIPAllocatorsByFamily[family]
+		allocator, ok := rs.alloc.serviceIPAllocatorsByFamily[family]
 		if !ok {
 			// cluster was configured for dual stack, then single stack
 			klog.V(4).Infof("delete service. Not releasing ClusterIP:%v because IPFamily:%v is no longer configured on server", ip, family)
@@ -636,14 +625,14 @@ func (rs *REST) allocServiceClusterIP(service *api.Service) (map[api.IPFamily]st
 	toAlloc := make(map[api.IPFamily]string)
 
 	// get clusterIP.. empty string if user did not specify an ip
-	toAlloc[rs.defaultServiceIPFamily] = service.Spec.ClusterIP
+	toAlloc[rs.alloc.defaultServiceIPFamily] = service.Spec.ClusterIP
 	// alloc
 	allocated, err := rs.allocClusterIPs(service, toAlloc)
 
 	// set
 	if err == nil {
-		service.Spec.ClusterIP = allocated[rs.defaultServiceIPFamily]
-		service.Spec.ClusterIPs = []string{allocated[rs.defaultServiceIPFamily]}
+		service.Spec.ClusterIP = allocated[rs.alloc.defaultServiceIPFamily]
+		service.Spec.ClusterIPs = []string{allocated[rs.alloc.defaultServiceIPFamily]}
 	}
 
 	return allocated, err
@@ -752,7 +741,7 @@ func (rs *REST) handleClusterIPsForUpdatedService(oldService *api.Service, servi
 		toRelease = make(map[api.IPFamily]string)
 		if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
 			// for non dual stack enabled cluster we use clusterIPs
-			toRelease[rs.defaultServiceIPFamily] = oldService.Spec.ClusterIP
+			toRelease[rs.alloc.defaultServiceIPFamily] = oldService.Spec.ClusterIP
 		} else {
 			// dual stack is enabled, collect ClusterIPs by families
 			for i, family := range oldService.Spec.IPFamilies {
@@ -963,14 +952,14 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 		if i >= len(service.Spec.IPFamilies) {
 			if isIPv6 {
 				// first make sure that family(ip) is configured
-				if _, found := rs.serviceIPAllocatorsByFamily[api.IPv6Protocol]; !found {
+				if _, found := rs.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol]; !found {
 					el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv6 on a cluster which is not configured for it")}
 					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
 				}
 				service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
 			} else {
 				// first make sure that family(ip) is configured
-				if _, found := rs.serviceIPAllocatorsByFamily[api.IPv4Protocol]; !found {
+				if _, found := rs.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol]; !found {
 					el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv4 on a cluster which is not configured for it")}
 					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
 				}
@@ -997,7 +986,7 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 		// If IPFamilies was not set by the user, start with the default
 		// family.
 		if len(service.Spec.IPFamilies) == 0 {
-			service.Spec.IPFamilies = []api.IPFamily{rs.defaultServiceIPFamily}
+			service.Spec.IPFamilies = []api.IPFamily{rs.alloc.defaultServiceIPFamily}
 		}
 
 		// this follows headful services. With one exception on a single stack
@@ -1024,13 +1013,13 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 
 	// asking for dual stack on a non dual stack cluster
 	// should fail without assigning any family
-	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack && len(rs.serviceIPAllocatorsByFamily) < 2 {
+	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack && len(rs.alloc.serviceIPAllocatorsByFamily) < 2 {
 		el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "Cluster is not configured for dual stack services"))
 	}
 
 	// if there is a family requested then it has to be configured on cluster
 	for i, ipFamily := range service.Spec.IPFamilies {
-		if _, found := rs.serviceIPAllocatorsByFamily[ipFamily]; !found {
+		if _, found := rs.alloc.serviceIPAllocatorsByFamily[ipFamily]; !found {
 			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilies").Index(i), service.Spec.ClusterIPs, fmt.Sprintf("ipfamily %v is not configured on cluster", ipFamily)))
 		}
 	}
@@ -1049,14 +1038,14 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 
 	// nil families, gets cluster default (if feature flag is not in effect, the strategy will take care of removing it)
 	if len(service.Spec.IPFamilies) == 0 {
-		service.Spec.IPFamilies = []api.IPFamily{rs.defaultServiceIPFamily}
+		service.Spec.IPFamilies = []api.IPFamily{rs.alloc.defaultServiceIPFamily}
 	}
 
 	// is this service looking for dual stack, and this cluster does have two families?
 	// if so, then append the missing family
 	if *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack &&
 		len(service.Spec.IPFamilies) == 1 &&
-		len(rs.serviceIPAllocatorsByFamily) == 2 {
+		len(rs.alloc.serviceIPAllocatorsByFamily) == 2 {
 
 		if service.Spec.IPFamilies[0] == api.IPv4Protocol {
 			service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index c5ef587e006..1c2a9b59add 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -240,7 +240,13 @@ func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Po
 		t.Fatalf("cannot create port allocator %v", err)
 	}
 
-	rest, _ := NewREST(serviceStorage, endpointStorage, podStorage.Pod, rPrimary, rSecondary, portAllocator, nil)
+	ipAllocators := map[api.IPFamily]ipallocator.Interface{
+		rPrimary.IPFamily(): rPrimary,
+	}
+	if rSecondary != nil {
+		ipAllocators[rSecondary.IPFamily()] = rSecondary
+	}
+	rest, _ := NewREST(serviceStorage, endpointStorage, podStorage.Pod, rPrimary.IPFamily(), ipAllocators, portAllocator, nil)
 
 	return rest, server
 }
@@ -332,7 +338,7 @@ func TestServiceRegistryCreate(t *testing.T) {
 			}
 
 			for i, family := range createdService.Spec.IPFamilies {
-				allocator := storage.serviceIPAllocatorsByFamily[family]
+				allocator := storage.alloc.serviceIPAllocatorsByFamily[family]
 				c := allocator.CIDR()
 				cidr := &c
 				if !cidr.Contains(netutils.ParseIPSloppy(createdService.Spec.ClusterIPs[i])) {
@@ -399,7 +405,7 @@ func TestServiceRegistryCreateDryRun(t *testing.T) {
 			}
 
 			for i, family := range tc.svc.Spec.IPFamilies {
-				alloc := storage.serviceIPAllocatorsByFamily[family]
+				alloc := storage.alloc.serviceIPAllocatorsByFamily[family]
 				if ipIsAllocated(t, alloc, tc.svc.Spec.ClusterIPs[i]) {
 					t.Errorf("unexpected side effect: ip allocated %v", tc.svc.Spec.ClusterIPs[i])
 				}
@@ -429,7 +435,7 @@ func TestDryRunNodePort(t *testing.T) {
 	if createdSvc.Spec.Ports[0].NodePort == 0 {
 		t.Errorf("expected NodePort value assigned")
 	}
-	if portIsAllocated(t, storage.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
+	if portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
 		t.Errorf("unexpected side effect: NodePort allocated")
 	}
 	_, err = getService(storage, ctx, svc.Name, &metav1.GetOptions{})
@@ -456,7 +462,7 @@ func TestDryRunNodePort(t *testing.T) {
 		t.Errorf("Expected %v, but got %v", expectNodePorts, actualNodePorts)
 	}
 	for i := range svc.Spec.Ports {
-		if portIsAllocated(t, storage.serviceNodePorts, svc.Spec.Ports[i].NodePort) {
+		if portIsAllocated(t, storage.alloc.serviceNodePorts, svc.Spec.Ports[i].NodePort) {
 			t.Errorf("unexpected side effect: NodePort allocated")
 		}
 	}
@@ -560,7 +566,7 @@ func TestServiceRegistryCreateMultiNodePortsService(t *testing.T) {
 		for i := range serviceNodePorts {
 			nodePort := serviceNodePorts[i]
 			// Release the node port at the end of the test case.
-			storage.serviceNodePorts.Release(nodePort)
+			storage.alloc.serviceNodePorts.Release(nodePort)
 		}
 	}
 }
@@ -902,7 +908,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) {
 	if created {
 		t.Errorf("expected not created")
 	}
-	if portIsAllocated(t, storage.serviceNodePorts, new1.Spec.Ports[0].NodePort) {
+	if portIsAllocated(t, storage.alloc.serviceNodePorts, new1.Spec.Ports[0].NodePort) {
 		t.Errorf("unexpected side effect: NodePort allocated")
 	}
 
@@ -915,7 +921,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Expected no error: %v", err)
 	}
-	if ipIsAllocated(t, storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily], new2.Spec.ClusterIP) {
+	if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], new2.Spec.ClusterIP) {
 		t.Errorf("unexpected side effect: ip allocated")
 	}
 
@@ -925,10 +931,10 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) {
 		t.Fatalf("Expected no error: %v", err)
 	}
 	svc = obj.(*api.Service)
-	if !ipIsAllocated(t, storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily], svc.Spec.ClusterIP) {
+	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], svc.Spec.ClusterIP) {
 		t.Errorf("expected IP to be allocated")
 	}
-	if !portIsAllocated(t, storage.serviceNodePorts, svc.Spec.Ports[0].NodePort) {
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, svc.Spec.Ports[0].NodePort) {
 		t.Errorf("expected NodePort to be allocated")
 	}
 
@@ -939,7 +945,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Expected no error: %v", err)
 	}
-	if !portIsAllocated(t, storage.serviceNodePorts, svc.Spec.Ports[0].NodePort) {
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, svc.Spec.Ports[0].NodePort) {
 		t.Errorf("unexpected side effect: NodePort unallocated")
 	}
 
@@ -957,7 +963,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) {
 	if err != nil {
 		t.Fatalf("expected no error: %v", err)
 	}
-	if !ipIsAllocated(t, storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily], svc.Spec.ClusterIP) {
+	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], svc.Spec.ClusterIP) {
 		t.Errorf("unexpected side effect: ip unallocated")
 	}
 }
@@ -1130,14 +1136,14 @@ func TestServiceRegistryDeleteDryRun(t *testing.T) {
 	if createdSvc.Spec.ClusterIP == "" {
 		t.Fatalf("expected ClusterIP to be set")
 	}
-	if !ipIsAllocated(t, storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily], createdSvc.Spec.ClusterIP) {
+	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], createdSvc.Spec.ClusterIP) {
 		t.Errorf("expected ClusterIP to be allocated")
 	}
 	_, _, err = storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}})
 	if err != nil {
 		t.Fatalf("Expected no error: %v", err)
 	}
-	if !ipIsAllocated(t, storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily], createdSvc.Spec.ClusterIP) {
+	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], createdSvc.Spec.ClusterIP) {
 		t.Errorf("unexpected side effect: ip unallocated")
 	}
 
@@ -1151,7 +1157,7 @@ func TestServiceRegistryDeleteDryRun(t *testing.T) {
 	if createdSvc.Spec.Ports[0].NodePort == 0 {
 		t.Fatalf("expected NodePort to be set")
 	}
-	if !portIsAllocated(t, storage.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
 		t.Errorf("expected NodePort to be allocated")
 	}
 
@@ -1161,7 +1167,7 @@ func TestServiceRegistryDeleteDryRun(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Expected no error: %v", err)
 	}
-	if !portIsAllocated(t, storage.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
 		t.Errorf("unexpected side effect: NodePort unallocated")
 	}
 }
@@ -1189,7 +1195,7 @@ func TestDualStackServiceRegistryDeleteDryRun(t *testing.T) {
 		t.Fatalf("Expected no error: %v", err)
 	}
 	for i, family := range dualstack_svc.Spec.IPFamilies {
-		if !ipIsAllocated(t, dualstack_storage.serviceIPAllocatorsByFamily[family], dualstack_svc.Spec.ClusterIPs[i]) {
+		if !ipIsAllocated(t, dualstack_storage.alloc.serviceIPAllocatorsByFamily[family], dualstack_svc.Spec.ClusterIPs[i]) {
 			t.Errorf("unexpected side effect: ip unallocated %v", dualstack_svc.Spec.ClusterIPs[i])
 		}
 	}
@@ -1492,7 +1498,7 @@ func TestServiceRegistryIPAllocation(t *testing.T) {
 	testIPs := []string{"1.2.3.93", "1.2.3.94", "1.2.3.95", "1.2.3.96"}
 	testIP := "not-an-ip"
 	for _, ip := range testIPs {
-		if !ipIsAllocated(t, storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily].(*ipallocator.Range), ip) {
+		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily].(*ipallocator.Range), ip) {
 			testIP = ip
 			break
 		}
@@ -1581,7 +1587,7 @@ func TestServiceRegistryIPUpdate(t *testing.T) {
 	testIPs := []string{"1.2.3.93", "1.2.3.94", "1.2.3.95", "1.2.3.96"}
 	testIP := ""
 	for _, ip := range testIPs {
-		if !ipIsAllocated(t, storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily].(*ipallocator.Range), ip) {
+		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily].(*ipallocator.Range), ip) {
 			testIP = ip
 			break
 		}
@@ -1955,7 +1961,7 @@ func TestInitClusterIP(t *testing.T) {
 
 			// pre allocate ips if any
 			for family, ip := range test.preAllocateClusterIPs {
-				allocator, ok := storage.serviceIPAllocatorsByFamily[family]
+				allocator, ok := storage.alloc.serviceIPAllocatorsByFamily[family]
 				if !ok {
 					t.Fatalf("test is incorrect, allocator does not exist on rest")
 				}
@@ -1986,7 +1992,7 @@ func TestInitClusterIP(t *testing.T) {
 					if netutils.IsIPv6String(ip) {
 						family = api.IPv6Protocol
 					}
-					allocator := storage.serviceIPAllocatorsByFamily[family]
+					allocator := storage.alloc.serviceIPAllocatorsByFamily[family]
 					if !ipIsAllocated(t, allocator, ip) {
 						t.Fatalf("expected ip:%v to be allocated by %v allocator. it was not", ip, family)
 					}
@@ -2021,7 +2027,7 @@ func TestInitClusterIP(t *testing.T) {
 				return
 			}
 
-			shouldUpgrade := len(newSvc.Spec.IPFamilies) == 2 && *(newSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.serviceIPAllocatorsByFamily) == 2
+			shouldUpgrade := len(newSvc.Spec.IPFamilies) == 2 && *(newSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.alloc.serviceIPAllocatorsByFamily) == 2
 			if shouldUpgrade && len(newSvc.Spec.ClusterIPs) < 2 {
 				t.Fatalf("Service should have been upgraded %+v", newSvc)
 			}
@@ -2037,7 +2043,7 @@ func TestInitClusterIP(t *testing.T) {
 func TestInitNodePorts(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
-	nodePortOp := portallocator.StartOperation(storage.serviceNodePorts, false)
+	nodePortOp := portallocator.StartOperation(storage.alloc.serviceNodePorts, false)
 
 	testCases := []struct {
 		name                     string
@@ -2103,7 +2109,7 @@ func TestInitNodePorts(t *testing.T) {
 		serviceNodePorts := collectServiceNodePorts(test.service)
 		if len(test.expectSpecifiedNodePorts) == 0 {
 			for _, nodePort := range serviceNodePorts {
-				if !storage.serviceNodePorts.Has(nodePort) {
+				if !storage.alloc.serviceNodePorts.Has(nodePort) {
 					t.Errorf("%q: unexpected NodePort %d, out of range", test.name, nodePort)
 				}
 			}
@@ -2113,7 +2119,7 @@ func TestInitNodePorts(t *testing.T) {
 		for i := range serviceNodePorts {
 			nodePort := serviceNodePorts[i]
 			// Release the node port at the end of the test case.
-			storage.serviceNodePorts.Release(nodePort)
+			storage.alloc.serviceNodePorts.Release(nodePort)
 		}
 	}
 }
@@ -2121,7 +2127,7 @@ func TestInitNodePorts(t *testing.T) {
 func TestUpdateNodePorts(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
-	nodePortOp := portallocator.StartOperation(storage.serviceNodePorts, false)
+	nodePortOp := portallocator.StartOperation(storage.alloc.serviceNodePorts, false)
 
 	testCases := []struct {
 		name                     string
@@ -2218,7 +2224,7 @@ func TestUpdateNodePorts(t *testing.T) {
 		serviceNodePorts := collectServiceNodePorts(test.newService)
 		if len(test.expectSpecifiedNodePorts) == 0 {
 			for _, nodePort := range serviceNodePorts {
-				if !storage.serviceNodePorts.Has(nodePort) {
+				if !storage.alloc.serviceNodePorts.Has(nodePort) {
 					t.Errorf("%q: unexpected NodePort %d, out of range", test.name, nodePort)
 				}
 			}
@@ -2228,7 +2234,7 @@ func TestUpdateNodePorts(t *testing.T) {
 		for i := range serviceNodePorts {
 			nodePort := serviceNodePorts[i]
 			// Release the node port at the end of the test case.
-			storage.serviceNodePorts.Release(nodePort)
+			storage.alloc.serviceNodePorts.Release(nodePort)
 		}
 	}
 }
@@ -2386,7 +2392,7 @@ func TestServiceUpgrade(t *testing.T) {
 			createdSvc := obj.(*api.Service)
 			// allocated IP
 			for family, ip := range testCase.allocateIPsBeforeUpdate {
-				alloc := storage.serviceIPAllocatorsByFamily[family]
+				alloc := storage.alloc.serviceIPAllocatorsByFamily[family]
 				if err := alloc.Allocate(netutils.ParseIPSloppy(ip)); err != nil {
 					t.Fatalf("test is incorrect, unable to preallocate ip:%v", ip)
 				}
@@ -2418,7 +2424,7 @@ func TestServiceUpgrade(t *testing.T) {
 			updatedSvc := updated.(*api.Service)
 			isValidClusterIPFields(t, storage, updatedSvc, updatedSvc)
 
-			shouldUpgrade := len(createdSvc.Spec.IPFamilies) == 2 && *(createdSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.serviceIPAllocatorsByFamily) == 2
+			shouldUpgrade := len(createdSvc.Spec.IPFamilies) == 2 && *(createdSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.alloc.serviceIPAllocatorsByFamily) == 2
 			if shouldUpgrade && len(updatedSvc.Spec.ClusterIPs) < 2 {
 				t.Fatalf("Service should have been upgraded %+v", createdSvc)
 			}
@@ -2430,7 +2436,7 @@ func TestServiceUpgrade(t *testing.T) {
 			// make sure that ips were allocated, correctly
 			for i, family := range updatedSvc.Spec.IPFamilies {
 				ip := updatedSvc.Spec.ClusterIPs[i]
-				allocator := storage.serviceIPAllocatorsByFamily[family]
+				allocator := storage.alloc.serviceIPAllocatorsByFamily[family]
 				if !ipIsAllocated(t, allocator, ip) {
 					t.Fatalf("expected ip:%v to be allocated by %v allocator. it was not", ip, family)
 				}
@@ -2562,7 +2568,7 @@ func TestServiceDowngrade(t *testing.T) {
 			if shouldDowngrade {
 				releasedIP := copySvc.Spec.ClusterIPs[1]
 				releasedIPFamily := copySvc.Spec.IPFamilies[1]
-				allocator := storage.serviceIPAllocatorsByFamily[releasedIPFamily]
+				allocator := storage.alloc.serviceIPAllocatorsByFamily[releasedIPFamily]
 
 				if ipIsAllocated(t, allocator, releasedIP) {
 					t.Fatalf("expected ip:%v to be released by %v allocator. it was not", releasedIP, releasedIPFamily)
@@ -2579,28 +2585,28 @@ func TestDefaultingValidation(t *testing.T) {
 
 	// takes in REST and modify it for a specific config
 	fnMakeSingleStackIPv4Allocator := func(rest *REST) {
-		rest.defaultServiceIPFamily = api.IPv4Protocol
-		rest.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv4Protocol: rest.serviceIPAllocatorsByFamily[api.IPv4Protocol]}
+		rest.alloc.defaultServiceIPFamily = api.IPv4Protocol
+		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv4Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol]}
 	}
 
 	fnMakeSingleStackIPv6Allocator := func(rest *REST) {
-		rest.defaultServiceIPFamily = api.IPv6Protocol
-		rest.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv6Protocol: rest.serviceIPAllocatorsByFamily[api.IPv6Protocol]}
+		rest.alloc.defaultServiceIPFamily = api.IPv6Protocol
+		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv6Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol]}
 	}
 
 	fnMakeDualStackStackIPv4IPv6Allocator := func(rest *REST) {
-		rest.defaultServiceIPFamily = api.IPv4Protocol
-		rest.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{
-			api.IPv6Protocol: rest.serviceIPAllocatorsByFamily[api.IPv6Protocol],
-			api.IPv4Protocol: rest.serviceIPAllocatorsByFamily[api.IPv4Protocol],
+		rest.alloc.defaultServiceIPFamily = api.IPv4Protocol
+		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{
+			api.IPv6Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol],
+			api.IPv4Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol],
 		}
 	}
 
 	fnMakeDualStackStackIPv6IPv4Allocator := func(rest *REST) {
-		rest.defaultServiceIPFamily = api.IPv6Protocol
-		rest.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{
-			api.IPv6Protocol: rest.serviceIPAllocatorsByFamily[api.IPv6Protocol],
-			api.IPv4Protocol: rest.serviceIPAllocatorsByFamily[api.IPv4Protocol],
+		rest.alloc.defaultServiceIPFamily = api.IPv6Protocol
+		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{
+			api.IPv6Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol],
+			api.IPv4Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol],
 		}
 	}
 

From b76a8c3c40af5b590e91f23c37a87b0c984b9f75 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 17 Nov 2020 21:48:46 -0800
Subject: [PATCH 03/79] Svc REST: move allocator methods -> alloc object

Move all allocator-related methods onto the alloc object so it can be
used in either REST layer.  There's an INORDINATE amount of test code
here and I am skeptical that it is all useful.  That's for later
commits.
---
 pkg/registry/core/service/storage/rest.go     | 86 +++++++++----------
 .../core/service/storage/rest_test.go         |  2 +-
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 1af49f057e2..093232c8692 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -193,7 +193,7 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 
 	// TODO: this should probably move to strategy.PrepareForCreate()
 	defer func() {
-		released, err := rs.releaseClusterIPs(toReleaseClusterIPs)
+		released, err := rs.alloc.releaseClusterIPs(toReleaseClusterIPs)
 		if err != nil {
 			klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v",
 				service.Name, toReleaseClusterIPs, released, err)
@@ -203,13 +203,13 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 	// try set ip families (for missing ip families)
 	// we do it here, since we want this to be visible
 	// even when dryRun == true
-	if err := rs.tryDefaultValidateServiceClusterIPFields(nil, service); err != nil {
+	if err := rs.alloc.tryDefaultValidateServiceClusterIPFields(nil, service); err != nil {
 		return nil, err
 	}
 
 	var err error
 	if !dryrun.IsDryRun(options.DryRun) {
-		toReleaseClusterIPs, err = rs.allocServiceClusterIPs(service)
+		toReleaseClusterIPs, err = rs.alloc.allocServiceClusterIPs(service)
 		if err != nil {
 			return nil, err
 		}
@@ -283,7 +283,7 @@ func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.Val
 			return nil, false, err
 		}
 
-		rs.releaseAllocatedResources(svc)
+		rs.alloc.releaseAllocatedResources(svc)
 	}
 
 	// TODO: this is duplicated from the generic storage, when this wrapper is fully removed we can drop this
@@ -299,11 +299,11 @@ func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.Val
 	return status, true, nil
 }
 
-func (rs *REST) releaseAllocatedResources(svc *api.Service) {
-	rs.releaseServiceClusterIPs(svc)
+func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
+	al.releaseServiceClusterIPs(svc)
 
 	for _, nodePort := range collectServiceNodePorts(svc) {
-		err := rs.alloc.serviceNodePorts.Release(nodePort)
+		err := al.serviceNodePorts.Release(nodePort)
 		if err != nil {
 			// these should be caught by an eventual reconciliation / restart
 			utilruntime.HandleError(fmt.Errorf("Error releasing service %s node port %d: %v", svc.Name, nodePort, err))
@@ -313,7 +313,7 @@ func (rs *REST) releaseAllocatedResources(svc *api.Service) {
 	if apiservice.NeedsHealthCheck(svc) {
 		nodePort := svc.Spec.HealthCheckNodePort
 		if nodePort > 0 {
-			err := rs.alloc.serviceNodePorts.Release(int(nodePort))
+			err := al.serviceNodePorts.Release(int(nodePort))
 			if err != nil {
 				// these should be caught by an eventual reconciliation / restart
 				utilruntime.HandleError(fmt.Errorf("Error releasing service %s health check node port %d: %v", svc.Name, nodePort, err))
@@ -337,7 +337,7 @@ func shouldAllocateNodePorts(service *api.Service) bool {
 
 // healthCheckNodePortUpdate handles HealthCheckNodePort allocation/release
 // and adjusts HealthCheckNodePort during service update if needed.
-func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
+func (al *RESTAllocStuff) healthCheckNodePortUpdate(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
 	neededHealthCheckNodePort := apiservice.NeedsHealthCheck(oldService)
 	oldHealthCheckNodePort := oldService.Spec.HealthCheckNodePort
 
@@ -420,13 +420,13 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	// on success: any ip that should be released, will  be released
 	defer func() {
 		// release the allocated, this is expected to be cleared if the entire function ran to success
-		if allocated_released, err := rs.releaseClusterIPs(allocated); err != nil {
+		if allocated_released, err := rs.alloc.releaseClusterIPs(allocated); err != nil {
 			klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v", service.Namespace, service.Name, err, allocated, allocated_released)
 
 		}
 		// performRelease is set when the enture function ran to success
 		if performRelease {
-			if toReleaseIPs_released, err := rs.releaseClusterIPs(toReleaseIPs); err != nil {
+			if toReleaseIPs_released, err := rs.alloc.releaseClusterIPs(toReleaseIPs); err != nil {
 				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v", service.Namespace, service.Name, err, toReleaseIPs, toReleaseIPs_released)
 			}
 		}
@@ -436,12 +436,12 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	defer nodePortOp.Finish()
 
 	// try set ip families (for missing ip families)
-	if err := rs.tryDefaultValidateServiceClusterIPFields(oldService, service); err != nil {
+	if err := rs.alloc.tryDefaultValidateServiceClusterIPFields(oldService, service); err != nil {
 		return nil, false, err
 	}
 
 	if !dryrun.IsDryRun(options.DryRun) {
-		allocated, toReleaseIPs, err = rs.handleClusterIPsForUpdatedService(oldService, service)
+		allocated, toReleaseIPs, err = rs.alloc.handleClusterIPsForUpdatedService(oldService, service)
 		if err != nil {
 			return nil, false, err
 		}
@@ -464,7 +464,7 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	}
 
 	// Handle ExternalTraffic related updates.
-	success, err := rs.healthCheckNodePortUpdate(oldService, service, nodePortOp)
+	success, err := rs.alloc.healthCheckNodePortUpdate(oldService, service, nodePortOp)
 	if !success || err != nil {
 		return nil, false, err
 	}
@@ -571,11 +571,11 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO
 	return r.services.ConvertToTable(ctx, object, tableOptions)
 }
 
-func (rs *REST) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]string) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]string) (map[api.IPFamily]string, error) {
 	allocated := make(map[api.IPFamily]string)
 
 	for family, ip := range toAlloc {
-		allocator := rs.alloc.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
+		allocator := al.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
 		if ip == "" {
 			allocatedIP, err := allocator.AllocateNext()
 			if err != nil {
@@ -595,14 +595,14 @@ func (rs *REST) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]s
 }
 
 // releases clusterIPs per family
-func (rs *REST) releaseClusterIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) releaseClusterIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
 	if toRelease == nil {
 		return nil, nil
 	}
 
 	released := make(map[api.IPFamily]string)
 	for family, ip := range toRelease {
-		allocator, ok := rs.alloc.serviceIPAllocatorsByFamily[family]
+		allocator, ok := al.serviceIPAllocatorsByFamily[family]
 		if !ok {
 			// cluster was configured for dual stack, then single stack
 			klog.V(4).Infof("delete service. Not releasing ClusterIP:%v because IPFamily:%v is no longer configured on server", ip, family)
@@ -621,25 +621,25 @@ func (rs *REST) releaseClusterIPs(toRelease map[api.IPFamily]string) (map[api.IP
 
 // standard allocator for dualstackgate==Off, hard wired dependency
 // and ignores policy, families and clusterIPs
-func (rs *REST) allocServiceClusterIP(service *api.Service) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service) (map[api.IPFamily]string, error) {
 	toAlloc := make(map[api.IPFamily]string)
 
 	// get clusterIP.. empty string if user did not specify an ip
-	toAlloc[rs.alloc.defaultServiceIPFamily] = service.Spec.ClusterIP
+	toAlloc[al.defaultServiceIPFamily] = service.Spec.ClusterIP
 	// alloc
-	allocated, err := rs.allocClusterIPs(service, toAlloc)
+	allocated, err := al.allocClusterIPs(service, toAlloc)
 
 	// set
 	if err == nil {
-		service.Spec.ClusterIP = allocated[rs.alloc.defaultServiceIPFamily]
-		service.Spec.ClusterIPs = []string{allocated[rs.alloc.defaultServiceIPFamily]}
+		service.Spec.ClusterIP = allocated[al.defaultServiceIPFamily]
+		service.Spec.ClusterIPs = []string{allocated[al.defaultServiceIPFamily]}
 	}
 
 	return allocated, err
 }
 
 // allocates ClusterIPs for a service
-func (rs *REST) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]string, error) {
 	// external name don't get ClusterIPs
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil, nil
@@ -651,7 +651,7 @@ func (rs *REST) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]s
 	}
 
 	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return rs.allocServiceClusterIP(service)
+		return al.allocServiceClusterIP(service)
 	}
 
 	toAlloc := make(map[api.IPFamily]string)
@@ -674,7 +674,7 @@ func (rs *REST) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]s
 	}
 
 	// allocate
-	allocated, err := rs.allocClusterIPs(service, toAlloc)
+	allocated, err := al.allocClusterIPs(service, toAlloc)
 
 	// set if successful
 	if err == nil {
@@ -702,7 +702,7 @@ func (rs *REST) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]s
 // this func does not perform actual release of clusterIPs. it returns
 // a map[family]ip for the caller to release when everything else has
 // executed successfully
-func (rs *REST) handleClusterIPsForUpdatedService(oldService *api.Service, service *api.Service) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Service, service *api.Service) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
 
 	// We don't want to upgrade (add an IP) or downgrade (remove an IP)
 	// following a cluster downgrade/upgrade to/from dual-stackness
@@ -725,7 +725,7 @@ func (rs *REST) handleClusterIPsForUpdatedService(oldService *api.Service, servi
 	// CASE A:
 	// Update service from ExternalName to non-ExternalName, should initialize ClusterIP.
 	if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName {
-		allocated, err := rs.allocServiceClusterIPs(service)
+		allocated, err := al.allocServiceClusterIPs(service)
 		return allocated, nil, err
 	}
 
@@ -741,7 +741,7 @@ func (rs *REST) handleClusterIPsForUpdatedService(oldService *api.Service, servi
 		toRelease = make(map[api.IPFamily]string)
 		if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
 			// for non dual stack enabled cluster we use clusterIPs
-			toRelease[rs.alloc.defaultServiceIPFamily] = oldService.Spec.ClusterIP
+			toRelease[al.defaultServiceIPFamily] = oldService.Spec.ClusterIP
 		} else {
 			// dual stack is enabled, collect ClusterIPs by families
 			for i, family := range oldService.Spec.IPFamilies {
@@ -771,7 +771,7 @@ func (rs *REST) handleClusterIPsForUpdatedService(oldService *api.Service, servi
 		toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1]
 
 		// allocate
-		allocated, err := rs.allocClusterIPs(service, toAllocate)
+		allocated, err := al.allocClusterIPs(service, toAllocate)
 		// set if successful
 		if err == nil {
 			service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]]
@@ -792,7 +792,7 @@ func (rs *REST) handleClusterIPsForUpdatedService(oldService *api.Service, servi
 }
 
 // for pre dual stack (gate == off). Hardwired to ClusterIP and ignores all new fields
-func (rs *REST) releaseServiceClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) releaseServiceClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
 	toRelease := make(map[api.IPFamily]string)
 
 	// we need to do that to handle cases where allocator is no longer configured on
@@ -803,11 +803,11 @@ func (rs *REST) releaseServiceClusterIP(service *api.Service) (released map[api.
 		toRelease[api.IPv4Protocol] = service.Spec.ClusterIP
 	}
 
-	return rs.releaseClusterIPs(toRelease)
+	return al.releaseClusterIPs(toRelease)
 }
 
 // releases allocated ClusterIPs for service that is about to be deleted
-func (rs *REST) releaseServiceClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) releaseServiceClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
 	// external name don't get ClusterIPs
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil, nil
@@ -819,7 +819,7 @@ func (rs *REST) releaseServiceClusterIPs(service *api.Service) (released map[api
 	}
 
 	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return rs.releaseServiceClusterIP(service)
+		return al.releaseServiceClusterIP(service)
 	}
 
 	toRelease := make(map[api.IPFamily]string)
@@ -830,7 +830,7 @@ func (rs *REST) releaseServiceClusterIPs(service *api.Service) (released map[api
 			toRelease[api.IPv4Protocol] = ip
 		}
 	}
-	return rs.releaseClusterIPs(toRelease)
+	return al.releaseClusterIPs(toRelease)
 }
 
 // tests if two preferred dual-stack service have matching ClusterIPFields
@@ -897,7 +897,7 @@ func isMatchingPreferDualStackClusterIPFields(oldService, service *api.Service)
 
 // attempts to default service ip families according to cluster configuration
 // while ensuring that provided families are configured on cluster.
-func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *api.Service) error {
+func (al *RESTAllocStuff) tryDefaultValidateServiceClusterIPFields(oldService, service *api.Service) error {
 	// can not do anything here
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil
@@ -952,14 +952,14 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 		if i >= len(service.Spec.IPFamilies) {
 			if isIPv6 {
 				// first make sure that family(ip) is configured
-				if _, found := rs.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol]; !found {
+				if _, found := al.serviceIPAllocatorsByFamily[api.IPv6Protocol]; !found {
 					el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv6 on a cluster which is not configured for it")}
 					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
 				}
 				service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
 			} else {
 				// first make sure that family(ip) is configured
-				if _, found := rs.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol]; !found {
+				if _, found := al.serviceIPAllocatorsByFamily[api.IPv4Protocol]; !found {
 					el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv4 on a cluster which is not configured for it")}
 					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
 				}
@@ -986,7 +986,7 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 		// If IPFamilies was not set by the user, start with the default
 		// family.
 		if len(service.Spec.IPFamilies) == 0 {
-			service.Spec.IPFamilies = []api.IPFamily{rs.alloc.defaultServiceIPFamily}
+			service.Spec.IPFamilies = []api.IPFamily{al.defaultServiceIPFamily}
 		}
 
 		// this follows headful services. With one exception on a single stack
@@ -1013,13 +1013,13 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 
 	// asking for dual stack on a non dual stack cluster
 	// should fail without assigning any family
-	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack && len(rs.alloc.serviceIPAllocatorsByFamily) < 2 {
+	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack && len(al.serviceIPAllocatorsByFamily) < 2 {
 		el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "Cluster is not configured for dual stack services"))
 	}
 
 	// if there is a family requested then it has to be configured on cluster
 	for i, ipFamily := range service.Spec.IPFamilies {
-		if _, found := rs.alloc.serviceIPAllocatorsByFamily[ipFamily]; !found {
+		if _, found := al.serviceIPAllocatorsByFamily[ipFamily]; !found {
 			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilies").Index(i), service.Spec.ClusterIPs, fmt.Sprintf("ipfamily %v is not configured on cluster", ipFamily)))
 		}
 	}
@@ -1038,14 +1038,14 @@ func (rs *REST) tryDefaultValidateServiceClusterIPFields(oldService, service *ap
 
 	// nil families, gets cluster default (if feature flag is not in effect, the strategy will take care of removing it)
 	if len(service.Spec.IPFamilies) == 0 {
-		service.Spec.IPFamilies = []api.IPFamily{rs.alloc.defaultServiceIPFamily}
+		service.Spec.IPFamilies = []api.IPFamily{al.defaultServiceIPFamily}
 	}
 
 	// is this service looking for dual stack, and this cluster does have two families?
 	// if so, then append the missing family
 	if *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack &&
 		len(service.Spec.IPFamilies) == 1 &&
-		len(rs.alloc.serviceIPAllocatorsByFamily) == 2 {
+		len(al.serviceIPAllocatorsByFamily) == 2 {
 
 		if service.Spec.IPFamilies[0] == api.IPv4Protocol {
 			service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 1c2a9b59add..43eb8ef56fa 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -3710,7 +3710,7 @@ func TestDefaultingValidation(t *testing.T) {
 				testCase.modifyRest(storage)
 			}
 
-			err := storage.tryDefaultValidateServiceClusterIPFields(testCase.oldSvc, testCase.svc)
+			err := storage.alloc.tryDefaultValidateServiceClusterIPFields(testCase.oldSvc, testCase.svc)
 			if err != nil && !testCase.expectError {
 				t.Fatalf("error %v was not expected", err)
 			}

From 14d0571a5fc3612f50ce2986a10b583c17615592 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 17 Nov 2020 22:10:19 -0800
Subject: [PATCH 04/79] Svc REST: Don't call validation directly

The validation is called soon after anyway.
---
 pkg/apis/core/validation/validation.go      | 5 +++--
 pkg/apis/core/validation/validation_test.go | 3 ++-
 pkg/registry/core/service/storage/rest.go   | 7 -------
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go
index a072d7cee28..d5c66ce84c4 100644
--- a/pkg/apis/core/validation/validation.go
+++ b/pkg/apis/core/validation/validation.go
@@ -4455,6 +4455,7 @@ func ValidateService(service *core.Service) field.ErrorList {
 
 	// external traffic fields
 	allErrs = append(allErrs, validateServiceExternalTrafficFieldsValue(service)...)
+	allErrs = append(allErrs, validateServiceExternalTrafficFieldsCombination(service)...)
 
 	// internal traffic policy field
 	allErrs = append(allErrs, validateServiceInternalTrafficFieldsValue(service)...)
@@ -4545,11 +4546,11 @@ func validateServiceInternalTrafficFieldsValue(service *core.Service) field.Erro
 	return allErrs
 }
 
-// ValidateServiceExternalTrafficFieldsCombination validates if ExternalTrafficPolicy,
+// validateServiceExternalTrafficFieldsCombination validates if ExternalTrafficPolicy,
 // HealthCheckNodePort and Type combination are legal. For update, it should be called
 // after clearing externalTraffic related fields for the ease of transitioning between
 // different service types.
-func ValidateServiceExternalTrafficFieldsCombination(service *core.Service) field.ErrorList {
+func validateServiceExternalTrafficFieldsCombination(service *core.Service) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	if service.Spec.Type != core.ServiceTypeLoadBalancer &&
diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go
index 2ec590aef68..64182ef6ef4 100644
--- a/pkg/apis/core/validation/validation_test.go
+++ b/pkg/apis/core/validation/validation_test.go
@@ -12019,7 +12019,8 @@ func TestValidateServiceExternalTrafficFieldsCombination(t *testing.T) {
 	for _, tc := range testCases {
 		svc := makeValidService()
 		tc.tweakSvc(&svc)
-		errs := ValidateServiceExternalTrafficFieldsCombination(&svc)
+		// TODO: This test is probably insufficient for such a big function under test.
+		errs := validateServiceExternalTrafficFieldsCombination(&svc)
 		if len(errs) != tc.numErrs {
 			t.Errorf("Unexpected error list for case %q: %v", tc.name, errs.ToAggregate())
 		}
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 093232c8692..57d2573b2c2 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -40,7 +40,6 @@ import (
 	"k8s.io/klog/v2"
 	apiservice "k8s.io/kubernetes/pkg/api/service"
 	api "k8s.io/kubernetes/pkg/apis/core"
-	"k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/features"
 	registry "k8s.io/kubernetes/pkg/registry/core/service"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
@@ -230,9 +229,6 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 			return nil, errors.NewInternalError(err)
 		}
 	}
-	if errs := validation.ValidateServiceExternalTrafficFieldsCombination(service); len(errs) > 0 {
-		return nil, errors.NewInvalid(api.Kind("Service"), service.Name, errs)
-	}
 
 	out, err := rs.services.Create(ctx, service, createValidation, options)
 	if err != nil {
@@ -468,9 +464,6 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	if !success || err != nil {
 		return nil, false, err
 	}
-	if errs := validation.ValidateServiceExternalTrafficFieldsCombination(service); len(errs) > 0 {
-		return nil, false, errors.NewInvalid(api.Kind("Service"), service.Name, errs)
-	}
 
 	out, created, err := rs.services.Update(ctx, service.Name, rest.DefaultUpdatedObjectInfo(service), createValidation, updateValidation, forceAllowCreate, options)
 	if err == nil {

From 960b36b1243ff883eaad9087d49c888083b38300 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 26 Jun 2021 11:51:52 -0700
Subject: [PATCH 05/79] Svc REST: Add a transaction API

This will be used in upcoming commits, but for easier history and review
it is pretty stand-alone.
---
 .../core/service/storage/transaction.go       |  62 +++++++
 .../core/service/storage/transaction_test.go  | 167 ++++++++++++++++++
 2 files changed, 229 insertions(+)
 create mode 100644 pkg/registry/core/service/storage/transaction.go
 create mode 100644 pkg/registry/core/service/storage/transaction_test.go

diff --git a/pkg/registry/core/service/storage/transaction.go b/pkg/registry/core/service/storage/transaction.go
new file mode 100644
index 00000000000..417e57323bd
--- /dev/null
+++ b/pkg/registry/core/service/storage/transaction.go
@@ -0,0 +1,62 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+// transaction represents something that may need to be finalized on success or
+// failure of the larger transaction.
+type transaction interface {
+	// Commit tells the transaction to finalize any changes it may have
+	// pending.  This cannot fail, so errors must be handled internally.
+	Commit()
+
+	// Revert tells the transaction to abandon or undo any changes it may have
+	// pending.  This cannot fail, so errors must be handled internally.
+	Revert()
+}
+
+// metaTransaction is a collection of transactions.
+type metaTransaction []transaction
+
+func (mt metaTransaction) Commit() {
+	for _, t := range mt {
+		t.Commit()
+	}
+}
+
+func (mt metaTransaction) Revert() {
+	for _, t := range mt {
+		t.Revert()
+	}
+}
+
+// callbackTransaction is a transaction which calls arbitrary functions.
+type callbackTransaction struct {
+	commit func()
+	revert func()
+}
+
+func (cb callbackTransaction) Commit() {
+	if cb.commit != nil {
+		cb.commit()
+	}
+}
+
+func (cb callbackTransaction) Revert() {
+	if cb.revert != nil {
+		cb.revert()
+	}
+}
diff --git a/pkg/registry/core/service/storage/transaction_test.go b/pkg/registry/core/service/storage/transaction_test.go
new file mode 100644
index 00000000000..e4f4e3220f4
--- /dev/null
+++ b/pkg/registry/core/service/storage/transaction_test.go
@@ -0,0 +1,167 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"testing"
+)
+
+func Test_metaTransaction(t *testing.T) {
+	const initial = 10
+	var temp int
+
+	tests := []struct {
+		name  string
+		mt    metaTransaction
+		start int
+		want  int
+	}{{
+		name: "commit and revert match",
+		mt: metaTransaction{
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 1
+				},
+				revert: func() {
+					temp = temp - 1
+				},
+			},
+		},
+		want: 10,
+	}, {
+		name: "commit and revert match multiple times",
+		mt: metaTransaction{
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 1
+				},
+				revert: func() {
+					temp = temp - 1
+				},
+			},
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 2
+				},
+				revert: func() {
+					temp = temp - 2
+				},
+			},
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 3
+				},
+				revert: func() {
+					temp = temp - 3
+				},
+			},
+		},
+		want: 10,
+	}, {
+		name: "missing revert",
+		mt: metaTransaction{
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 1
+				},
+				revert: func() {
+					temp = temp - 1
+				},
+			},
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 2
+				},
+			},
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 3
+				},
+				revert: func() {
+					temp = temp - 3
+				},
+			},
+		},
+		want: 12,
+	}, {
+		name: "missing commit",
+		mt: metaTransaction{
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 1
+				},
+				revert: func() {
+					temp = temp - 1
+				},
+			},
+			callbackTransaction{
+				revert: func() {
+					temp = temp - 2
+				},
+			},
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 3
+				},
+				revert: func() {
+					temp = temp - 3
+				},
+			},
+		},
+		want: 8,
+	}, {
+		name: "commit and revert match multiple but different order",
+		mt: metaTransaction{
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 1
+				},
+				revert: func() {
+					temp = temp - 2
+				},
+			},
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 2
+				},
+				revert: func() {
+					temp = temp - 1
+				},
+			},
+			callbackTransaction{
+				commit: func() {
+					temp = temp + 3
+				},
+				revert: func() {
+					temp = temp - 3
+				},
+			},
+		},
+		want: 10,
+	}}
+	t.Parallel()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			temp = initial
+			tt.mt.Commit()
+			tt.mt.Revert()
+			if temp != tt.want {
+				t.Fatalf("expected %d got %d", tt.want, temp)
+			}
+		})
+	}
+}

From f3c7e846f1c3846090e9ba7c62892bcae46ca7d3 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 26 Jun 2021 11:55:30 -0700
Subject: [PATCH 06/79] Svc REST: Move allocations in Create into funcs

All the logic remains unchanged, just reorganized.  The functions are
imperfect but emphasize the change being made and can be cleaned up
subsequently.

This makes the following steps easier to comprehend.
---
 pkg/registry/core/service/storage/rest.go     | 151 ++++++++++++------
 .../core/service/storage/rest_test.go         |  12 +-
 2 files changed, 111 insertions(+), 52 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 57d2573b2c2..04d04057646 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -182,53 +182,24 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 
 	service := obj.(*api.Service)
 
-	// bag of clusterIPs allocated in the process of creation
-	// failed allocation will automatically trigger release
-	var toReleaseClusterIPs map[api.IPFamily]string
-
 	if err := rest.BeforeCreate(rs.strategy, ctx, obj); err != nil {
 		return nil, err
 	}
 
-	// TODO: this should probably move to strategy.PrepareForCreate()
-	defer func() {
-		released, err := rs.alloc.releaseClusterIPs(toReleaseClusterIPs)
-		if err != nil {
-			klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v",
-				service.Name, toReleaseClusterIPs, released, err)
-		}
-	}()
-
-	// try set ip families (for missing ip families)
-	// we do it here, since we want this to be visible
-	// even when dryRun == true
-	if err := rs.alloc.tryDefaultValidateServiceClusterIPFields(nil, service); err != nil {
+	// Allocate IPs and ports. If we had a transactional store, this would just
+	// be part of the larger transaction.  We don't have that, so we have to do
+	// it manually. This has to happen here and not in any earlier hooks (e.g.
+	// defaulting) because it needs to be aware of flags and be able to access
+	// API storage.
+	txn, err := rs.alloc.allocateCreate(service, dryrun.IsDryRun(options.DryRun))
+	if err != nil {
 		return nil, err
 	}
-
-	var err error
-	if !dryrun.IsDryRun(options.DryRun) {
-		toReleaseClusterIPs, err = rs.alloc.allocServiceClusterIPs(service)
-		if err != nil {
-			return nil, err
+	defer func() {
+		if txn != nil {
+			txn.Revert()
 		}
-	}
-
-	nodePortOp := portallocator.StartOperation(rs.alloc.serviceNodePorts, dryrun.IsDryRun(options.DryRun))
-	defer nodePortOp.Finish()
-
-	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
-		if err := initNodePorts(service, nodePortOp); err != nil {
-			return nil, err
-		}
-	}
-
-	// Handle ExternalTraffic related fields during service creation.
-	if apiservice.NeedsHealthCheck(service) {
-		if err := allocateHealthCheckNodePort(service, nodePortOp); err != nil {
-			return nil, errors.NewInternalError(err)
-		}
-	}
+	}()
 
 	out, err := rs.services.Create(ctx, service, createValidation, options)
 	if err != nil {
@@ -236,19 +207,45 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 	}
 
 	if err == nil {
-		el := nodePortOp.Commit()
-		if el != nil {
-			// these should be caught by an eventual reconciliation / restart
-			utilruntime.HandleError(fmt.Errorf("error(s) committing service node-ports changes: %v", el))
-		}
-
-		// no clusterips to release
-		toReleaseClusterIPs = nil
+		txn.Commit()
+		txn = nil
 	}
 
 	return out, err
 }
 
+func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (transaction, error) {
+	result := metaTransaction{}
+
+	// Ensure IP family fields are correctly initialized.  We do it here, since
+	// we want this to be visible even when dryRun == true.
+	if err := al.tryDefaultValidateServiceClusterIPFields(nil, service); err != nil {
+		return nil, err
+	}
+
+	// Allocate ClusterIPs
+	//FIXME: we need to put values in, even if dry run - else validation should
+	//not pass.  It does but that should be fixed.
+	if !dryRun {
+		if txn, err := al.allocServiceClusterIPsNew(service); err != nil {
+			result.Revert()
+			return nil, err
+		} else {
+			result = append(result, txn)
+		}
+	}
+
+	// Allocate ports
+	if txn, err := al.allocServiceNodePortsNew(service, dryRun); err != nil {
+		result.Revert()
+		return nil, err
+	} else {
+		result = append(result, txn)
+	}
+
+	return result, nil
+}
+
 func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
 	// TODO: handle graceful
 	obj, _, err := rs.services.Delete(ctx, id, deleteValidation, options)
@@ -631,6 +628,27 @@ func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service) (map[api.I
 	return allocated, err
 }
 
+//FIXME: merge into allocServiceClusterIPs rather than call it
+func (al *RESTAllocStuff) allocServiceClusterIPsNew(service *api.Service) (transaction, error) {
+	// clusterIPs that were allocated may need to be released in case of
+	// failure at a higher level.
+	toReleaseClusterIPs, err := al.allocServiceClusterIPs(service)
+	if err != nil {
+		return nil, err
+	}
+
+	txn := callbackTransaction{
+		revert: func() {
+			released, err := al.releaseClusterIPs(toReleaseClusterIPs)
+			if err != nil {
+				klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v",
+					service.Name, toReleaseClusterIPs, released, err)
+			}
+		},
+	}
+	return txn, nil
+}
+
 // allocates ClusterIPs for a service
 func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]string, error) {
 	// external name don't get ClusterIPs
@@ -1135,6 +1153,43 @@ func allocateHealthCheckNodePort(service *api.Service, nodePortOp *portallocator
 	return nil
 }
 
+//FIXME: rename and merge with initNodePorts
+func (al *RESTAllocStuff) allocServiceNodePortsNew(service *api.Service, dryRun bool) (transaction, error) {
+	// The allocator tracks dry-run-ness internally.
+	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
+
+	txn := callbackTransaction{
+		commit: func() {
+			nodePortOp.Commit()
+			// We don't NEED to call Finish() here, but for that package says
+			// to, so for future-safety, we will.
+			nodePortOp.Finish()
+		},
+		revert: func() {
+			// Weirdly named but this will revert if commit wasn't called
+			nodePortOp.Finish()
+		},
+	}
+
+	// Allocate NodePorts, if needed.
+	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
+		if err := initNodePorts(service, nodePortOp); err != nil {
+			txn.Revert()
+			return nil, err
+		}
+	}
+
+	// Handle ExternalTraffic related fields during service creation.
+	if apiservice.NeedsHealthCheck(service) {
+		if err := allocateHealthCheckNodePort(service, nodePortOp); err != nil {
+			txn.Revert()
+			return nil, errors.NewInternalError(err)
+		}
+	}
+
+	return txn, nil
+}
+
 func initNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
 	svcPortToNodePort := map[int]int{}
 	for i := range service.Spec.Ports {
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 43eb8ef56fa..096b0557809 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -1089,18 +1089,22 @@ func TestAllocateLoadBalancerNodePorts(t *testing.T) {
 				if tc.expectError {
 					return
 				}
-				t.Errorf("%s; Failed to create service: %#v", tc.name, err)
+				t.Errorf("failed to create service: %#v", err)
 			}
 			srv, err := getService(storage, ctx, tc.svc.Name, &metav1.GetOptions{})
 			if err != nil {
-				t.Errorf("%s; Unexpected error: %v", tc.name, err)
+				t.Errorf("unexpected error: %v", err)
 			}
 			if srv == nil {
-				t.Fatalf("%s; Failed to find service: %s", tc.name, tc.svc.Name)
+				t.Fatalf("failed to find service: %s", tc.svc.Name)
 			}
 			serviceNodePorts := collectServiceNodePorts(srv)
 			if (len(serviceNodePorts) != 0) != tc.expectNodePorts {
-				t.Errorf("%s; Allocated NodePorts not as expected", tc.name)
+				exp := "0"
+				if tc.expectNodePorts {
+					exp = ">0"
+				}
+				t.Errorf("allocated NodePorts not as expected: expected %v, got %v", exp, len(serviceNodePorts))
 			}
 		})
 	}

From 5e7e35ca45ba3112d1ebd4da9acd08d601ec4f45 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 17 Nov 2020 15:18:10 -0800
Subject: [PATCH 07/79] Svc REST: Add stub begin* hooks

These will be used in the next set of commits to de-0layer service REST.
---
 pkg/registry/core/service/storage/storage.go | 36 ++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 2ec4bd64389..ed8e5be0f35 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -89,6 +89,8 @@ func NewGenericREST(optsGetter generic.RESTOptionsGetter, serviceCIDR net.IPNet,
 	}
 	genericStore := &GenericREST{store, primaryIPFamily, secondaryFamily}
 	store.Decorator = genericStore.defaultOnRead
+	store.BeginCreate = genericStore.beginCreate
+	store.BeginUpdate = genericStore.beginUpdate
 
 	return genericStore, &StatusREST{store: &statusStore}, nil
 }
@@ -240,3 +242,37 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 		}
 	}
 }
+
+func (r *GenericREST) beginCreate(ctx context.Context, obj runtime.Object, options *metav1.CreateOptions) (genericregistry.FinishFunc, error) {
+	svc := obj.(*api.Service)
+
+	// FIXME: remove this when implementing
+	_ = svc
+
+	// Our cleanup callback
+	finish := func(_ context.Context, success bool) {
+		if success {
+		} else {
+		}
+	}
+
+	return finish, nil
+}
+
+func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Object, options *metav1.UpdateOptions) (genericregistry.FinishFunc, error) {
+	newSvc := obj.(*api.Service)
+	oldSvc := oldObj.(*api.Service)
+
+	// FIXME: remove these when implementing
+	_ = oldSvc
+	_ = newSvc
+
+	// Our cleanup callback
+	finish := func(_ context.Context, success bool) {
+		if success {
+		} else {
+		}
+	}
+
+	return finish, nil
+}

From 634055bded8734e9a98b1e0ff36fd3235defbe2e Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 17 Nov 2020 16:30:46 -0800
Subject: [PATCH 08/79] Svc REST: De-layer Create

Gut the "outer" Create() and move it to the inner BeginCreate().  This
uses a "transaction" type to make cleanup functions easy to read.

Background:

Service has an "outer" and "inner" REST handler.  This is because of how we do IP and port allocations synchronously, but since we don't have API transactions, we need to roll those back in case of a failure.  Both layers use the same `Strategy`, but the outer calls into the inner, which causes a lot of complexity in the code (including an open-coded partial reimplementation of a date-unknown snapshot of the generic REST code) and results in `Prepare` and `Validate` hooks being called twice.

The "normal" REST flow seems to be:

```
mutating webhooks
generic REST store Create {
    cleanup = BeginCreate
    BeforeCreate {
        strategy.PrepareForCreate {
            dropDisabledFields
        }
        strategy.Validate
        strategy.Canonicalize
    }
    createValidation (validating webhooks)
    storage Create
    cleanup
    AfterCreate
    Decorator
}
```

Service (before this commit) does:

```
mutating webhooks
svc custom Create {
    BeforeCreate {
        strategy.PrepareForCreate {
            dropDisabledFields
        }
        strategy.Validate
        strategy.Canonicalize
    }
    Allocations
    inner (generic) Create {
        cleanup = BeginCreate
        BeforeCreate {
            strategy.PrepareForCreate {
                dropDisabledFields
            }
            strategy.Validate
            strategy.Canonicalize
        }
        createValidation (validating webhooks)
        storage Create
        cleanup
        AfterCreate
        Decorator
    }
}
```

After this commit:

```
mutating webhooks
generic REST store Create {
    cleanup = BeginCreate
        Allocations
    BeforeCreate {
        strategy.PrepareForCreate {
            dropDisabledFields
        }
        strategy.Validate
        strategy.Canonicalize
    }
    createValidation (validating webhooks)
    storage Create
    cleanup
    AfterCreate
        Rollback allocations on error
    Decorator
}
```

This same fix pattern will be applied to Delete and Update in subsequent
commits.
---
 pkg/registry/core/rest/storage_core.go        |  2 +-
 pkg/registry/core/service/storage/rest.go     | 36 +--------
 .../core/service/storage/rest_test.go         | 36 +++++++--
 pkg/registry/core/service/storage/storage.go  | 81 ++++++++++++-------
 .../core/service/storage/storage_test.go      | 76 ++++++++++-------
 pkg/registry/core/service/strategy.go         |  3 +
 6 files changed, 130 insertions(+), 104 deletions(-)

diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go
index 9f5d7a668c5..d2fa6a1d45f 100644
--- a/pkg/registry/core/rest/storage_core.go
+++ b/pkg/registry/core/rest/storage_core.go
@@ -261,7 +261,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		serviceIPAllocators[secondaryServiceClusterIPAllocator.IPFamily()] = secondaryServiceClusterIPAllocator
 	}
 
-	serviceRESTStorage, serviceStatusStorage, err := servicestore.NewGenericREST(restOptionsGetter, serviceClusterIPRange, secondaryServiceClusterIPAllocator != nil)
+	serviceRESTStorage, serviceStatusStorage, err := servicestore.NewGenericREST(restOptionsGetter, serviceClusterIPAllocator.IPFamily(), serviceIPAllocators, serviceNodePortAllocator)
 	if err != nil {
 		return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err
 	}
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 04d04057646..52e8f199020 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -177,41 +177,7 @@ func (rs *REST) Watch(ctx context.Context, options *metainternalversion.ListOpti
 }
 
 func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
-	// DeepCopy to prevent writes here propagating back to tests.
-	obj = obj.DeepCopyObject()
-
-	service := obj.(*api.Service)
-
-	if err := rest.BeforeCreate(rs.strategy, ctx, obj); err != nil {
-		return nil, err
-	}
-
-	// Allocate IPs and ports. If we had a transactional store, this would just
-	// be part of the larger transaction.  We don't have that, so we have to do
-	// it manually. This has to happen here and not in any earlier hooks (e.g.
-	// defaulting) because it needs to be aware of flags and be able to access
-	// API storage.
-	txn, err := rs.alloc.allocateCreate(service, dryrun.IsDryRun(options.DryRun))
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if txn != nil {
-			txn.Revert()
-		}
-	}()
-
-	out, err := rs.services.Create(ctx, service, createValidation, options)
-	if err != nil {
-		err = rest.CheckGeneratedNameError(ctx, rs.strategy, err, service)
-	}
-
-	if err == nil {
-		txn.Commit()
-		txn = nil
-	}
-
-	return out, err
+	return rs.services.Create(ctx, obj, createValidation, options)
 }
 
 func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (transaction, error) {
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 096b0557809..9444a651d6a 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -37,6 +37,7 @@ import (
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/util/dryrun"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -60,6 +61,7 @@ import (
 // in a completely different way. We should unify it.
 
 type serviceStorage struct {
+	inner    *GenericREST
 	Services map[string]*api.Service
 }
 
@@ -118,12 +120,16 @@ func (s *serviceStorage) New() runtime.Object {
 }
 
 func (s *serviceStorage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
-	if dryrun.IsDryRun(options.DryRun) {
-		return obj, nil
+	ret, err := s.inner.Create(ctx, obj, createValidation, options)
+	if err != nil {
+		return ret, err
 	}
-	svc := obj.(*api.Service)
+
+	if dryrun.IsDryRun(options.DryRun) {
+		return ret.DeepCopyObject(), nil
+	}
+	svc := ret.(*api.Service)
 	s.saveService(svc)
-	s.Services[svc.Name].ResourceVersion = "1"
 
 	return s.Services[svc.Name].DeepCopy(), nil
 }
@@ -173,8 +179,6 @@ func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.
 func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Pod, ipFamilies []api.IPFamily) (*REST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 
-	serviceStorage := &serviceStorage{}
-
 	podStorage, err := podstore.NewStorage(generic.RESTOptions{
 		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "pods"}),
 		Decorator:               generic.UndecoratedStorage,
@@ -246,11 +250,29 @@ func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Po
 	if rSecondary != nil {
 		ipAllocators[rSecondary.IPFamily()] = rSecondary
 	}
-	rest, _ := NewREST(serviceStorage, endpointStorage, podStorage.Pod, rPrimary.IPFamily(), ipAllocators, portAllocator, nil)
+
+	inner := newInnerREST(t, etcdStorage, ipAllocators, portAllocator)
+	rest, _ := NewREST(inner, endpointStorage, podStorage.Pod, rPrimary.IPFamily(), ipAllocators, portAllocator, nil)
 
 	return rest, server
 }
 
+// This bridges to the "inner" REST implementation so tests continue to run
+// during the delayering of service REST code.
+func newInnerREST(t *testing.T, etcdStorage *storagebackend.ConfigForResource, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) *serviceStorage {
+	restOptions := generic.RESTOptions{
+		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
+		Decorator:               generic.UndecoratedStorage,
+		DeleteCollectionWorkers: 1,
+		ResourcePrefix:          "services",
+	}
+	inner, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, portAlloc)
+	if err != nil {
+		t.Fatalf("unexpected error from REST storage: %v", err)
+	}
+	return &serviceStorage{inner: inner}
+}
+
 func makeIPNet(t *testing.T) *net.IPNet {
 	_, net, err := netutils.ParseCIDRSloppy("1.2.3.0/24")
 	if err != nil {
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index ed8e5be0f35..6b0e1a81309 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -18,13 +18,13 @@ package storage
 
 import (
 	"context"
-	"net"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
 	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/apiserver/pkg/util/dryrun"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
@@ -32,8 +32,9 @@ import (
 	printersinternal "k8s.io/kubernetes/pkg/printers/internalversion"
 	printerstorage "k8s.io/kubernetes/pkg/printers/storage"
 	"k8s.io/kubernetes/pkg/registry/core/service"
-	registry "k8s.io/kubernetes/pkg/registry/core/service"
 	svcreg "k8s.io/kubernetes/pkg/registry/core/service"
+	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
+	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 
 	netutil "k8s.io/utils/net"
@@ -41,13 +42,19 @@ import (
 
 type GenericREST struct {
 	*genericregistry.Store
-	primaryIPFamily *api.IPFamily
-	secondaryFamily *api.IPFamily
+	primaryIPFamily   api.IPFamily
+	secondaryIPFamily api.IPFamily
+	alloc             RESTAllocStuff
 }
 
 // NewGenericREST returns a RESTStorage object that will work against services.
-func NewGenericREST(optsGetter generic.RESTOptionsGetter, serviceCIDR net.IPNet, hasSecondary bool) (*GenericREST, *StatusREST, error) {
-	strategy, _ := registry.StrategyForServiceCIDRs(serviceCIDR, hasSecondary)
+func NewGenericREST(
+	optsGetter generic.RESTOptionsGetter,
+	serviceIPFamily api.IPFamily,
+	ipAllocs map[api.IPFamily]ipallocator.Interface,
+	portAlloc portallocator.Interface) (*GenericREST, *StatusREST, error) {
+
+	strategy, _ := svcreg.StrategyForServiceCIDRs(ipAllocs[serviceIPFamily].CIDR(), len(ipAllocs) > 1)
 
 	store := &genericregistry.Store{
 		NewFunc:                  func() runtime.Object { return &api.Service{} },
@@ -72,22 +79,12 @@ func NewGenericREST(optsGetter generic.RESTOptionsGetter, serviceCIDR net.IPNet,
 	statusStore.UpdateStrategy = statusStrategy
 	statusStore.ResetFieldsStrategy = statusStrategy
 
-	ipv4 := api.IPv4Protocol
-	ipv6 := api.IPv6Protocol
-	var primaryIPFamily *api.IPFamily
-	var secondaryFamily *api.IPFamily
-	if netutil.IsIPv6CIDR(&serviceCIDR) {
-		primaryIPFamily = &ipv6
-		if hasSecondary {
-			secondaryFamily = &ipv4
-		}
-	} else {
-		primaryIPFamily = &ipv4
-		if hasSecondary {
-			secondaryFamily = &ipv6
-		}
+	var primaryIPFamily api.IPFamily = serviceIPFamily
+	var secondaryIPFamily api.IPFamily = "" // sentinel value
+	if len(ipAllocs) > 1 {
+		secondaryIPFamily = otherFamily(serviceIPFamily)
 	}
-	genericStore := &GenericREST{store, primaryIPFamily, secondaryFamily}
+	genericStore := &GenericREST{store, primaryIPFamily, secondaryIPFamily, makeAlloc(serviceIPFamily, ipAllocs, portAlloc)}
 	store.Decorator = genericStore.defaultOnRead
 	store.BeginCreate = genericStore.beginCreate
 	store.BeginUpdate = genericStore.beginUpdate
@@ -95,6 +92,15 @@ func NewGenericREST(optsGetter generic.RESTOptionsGetter, serviceCIDR net.IPNet,
 	return genericStore, &StatusREST{store: &statusStore}, nil
 }
 
+// otherFamily returns the non-selected IPFamily.  This assumes the input is
+// valid.
+func otherFamily(fam api.IPFamily) api.IPFamily {
+	if fam == api.IPv4Protocol {
+		return api.IPv6Protocol
+	}
+	return api.IPv4Protocol
+}
+
 var (
 	_ rest.ShortNamesProvider = &GenericREST{}
 	_ rest.CategoriesProvider = &GenericREST{}
@@ -196,7 +202,7 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 	preferDualStack := api.IPFamilyPolicyPreferDualStack
 	// headless services
 	if len(service.Spec.ClusterIPs) == 1 && service.Spec.ClusterIPs[0] == api.ClusterIPNone {
-		service.Spec.IPFamilies = []api.IPFamily{*r.primaryIPFamily}
+		service.Spec.IPFamilies = []api.IPFamily{r.primaryIPFamily}
 
 		// headless+selectorless
 		// headless+selectorless takes both families. Why?
@@ -205,7 +211,7 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 		// it to PreferDualStack on any cluster (single or dualstack configured).
 		if len(service.Spec.Selector) == 0 {
 			service.Spec.IPFamilyPolicy = &preferDualStack
-			if *r.primaryIPFamily == api.IPv4Protocol {
+			if r.primaryIPFamily == api.IPv4Protocol {
 				service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
 			} else {
 				service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol)
@@ -216,8 +222,8 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 			// selector and will have to follow how the cluster is configured. If the cluster is
 			// configured to dual stack then the service defaults to PreferDualStack. Otherwise we
 			// default it to SingleStack.
-			if r.secondaryFamily != nil {
-				service.Spec.IPFamilies = append(service.Spec.IPFamilies, *r.secondaryFamily)
+			if r.secondaryIPFamily != "" {
+				service.Spec.IPFamilies = append(service.Spec.IPFamilies, r.secondaryIPFamily)
 				service.Spec.IPFamilyPolicy = &preferDualStack
 			} else {
 				service.Spec.IPFamilyPolicy = &singleStack
@@ -246,13 +252,27 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 func (r *GenericREST) beginCreate(ctx context.Context, obj runtime.Object, options *metav1.CreateOptions) (genericregistry.FinishFunc, error) {
 	svc := obj.(*api.Service)
 
-	// FIXME: remove this when implementing
-	_ = svc
+	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
+	// early, before anyone looks at them.
+	// NOTE: the args are (old, new)
+	svcreg.NormalizeClusterIPs(nil, svc)
+
+	// Allocate IPs and ports. If we had a transactional store, this would just
+	// be part of the larger transaction.  We don't have that, so we have to do
+	// it manually. This has to happen here and not in any earlier hooks (e.g.
+	// defaulting) because it needs to be aware of flags and be able to access
+	// API storage.
+	txn, err := r.alloc.allocateCreate(svc, dryrun.IsDryRun(options.DryRun))
+	if err != nil {
+		return nil, err
+	}
 
 	// Our cleanup callback
 	finish := func(_ context.Context, success bool) {
 		if success {
+			txn.Commit()
 		} else {
+			txn.Revert()
 		}
 	}
 
@@ -263,9 +283,10 @@ func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Objec
 	newSvc := obj.(*api.Service)
 	oldSvc := oldObj.(*api.Service)
 
-	// FIXME: remove these when implementing
-	_ = oldSvc
-	_ = newSvc
+	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
+	// early, before anyone looks at them.
+	// NOTE: the args are (old, new)
+	svcreg.NormalizeClusterIPs(oldSvc, newSvc)
 
 	// Our cleanup callback
 	finish := func(_ context.Context, success bool) {
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 6fb4b76141d..43b72e14350 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -17,6 +17,8 @@ limitations under the License.
 package storage
 
 import (
+	"fmt"
+	"net"
 	"reflect"
 	"testing"
 
@@ -30,6 +32,7 @@ import (
 	genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing"
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
 	netutils "k8s.io/utils/net"
 
@@ -38,6 +41,14 @@ import (
 	"k8s.io/kubernetes/pkg/features"
 )
 
+func makeIPAllocator(cidr *net.IPNet) ipallocator.Interface {
+	al, err := ipallocator.NewInMemory(cidr)
+	if err != nil {
+		panic(fmt.Sprintf("error creating IP allocator: %v", err))
+	}
+	return al
+}
+
 func newStorage(t *testing.T) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 	restOptions := generic.RESTOptions{
@@ -46,7 +57,10 @@ func newStorage(t *testing.T) (*GenericREST, *StatusREST, *etcd3testing.EtcdTest
 		DeleteCollectionWorkers: 1,
 		ResourcePrefix:          "services",
 	}
-	serviceStorage, statusStorage, err := NewGenericREST(restOptions, *makeIPNet(t), false)
+	ipAllocs := map[api.IPFamily]ipallocator.Interface{
+		api.IPv4Protocol: makeIPAllocator(makeIPNet(t)),
+	}
+	serviceStorage, statusStorage, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, nil)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
@@ -427,7 +441,10 @@ func TestServiceDefaultOnRead(t *testing.T) {
 				t.Fatalf("failed to parse CIDR")
 			}
 
-			serviceStorage, _, err := NewGenericREST(restOptions, *cidr, false)
+			ipAllocs := map[api.IPFamily]ipallocator.Interface{
+				api.IPv4Protocol: makeIPAllocator(cidr),
+			}
+			serviceStorage, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, nil)
 			if err != nil {
 				t.Fatalf("unexpected error from REST storage: %v", err)
 			}
@@ -471,7 +488,7 @@ func TestServiceDefaultOnRead(t *testing.T) {
 }
 
 func TestServiceDefaulting(t *testing.T) {
-	makeStorage := func(t *testing.T, primaryCIDR string, isDualStack bool) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
+	makeStorage := func(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
 		etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 		restOptions := generic.RESTOptions{
 			StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
@@ -480,12 +497,19 @@ func TestServiceDefaulting(t *testing.T) {
 			ResourcePrefix:          "services",
 		}
 
-		_, cidr, err := netutils.ParseCIDRSloppy(primaryCIDR)
-		if err != nil {
-			t.Fatalf("failed to parse CIDR %s", primaryCIDR)
+		ipAllocs := map[api.IPFamily]ipallocator.Interface{}
+		for _, fam := range ipFamilies {
+			switch fam {
+			case api.IPv4Protocol:
+				_, cidr, _ := netutils.ParseCIDRSloppy("10.0.0.0/16")
+				ipAllocs[fam] = makeIPAllocator(cidr)
+			case api.IPv6Protocol:
+				_, cidr, _ := netutils.ParseCIDRSloppy("2000::/108")
+				ipAllocs[fam] = makeIPAllocator(cidr)
+			}
 		}
 
-		serviceStorage, statusStorage, err := NewGenericREST(restOptions, *(cidr), isDualStack)
+		serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, nil)
 		if err != nil {
 			t.Fatalf("unexpected error from REST storage: %v", err)
 		}
@@ -493,35 +517,25 @@ func TestServiceDefaulting(t *testing.T) {
 	}
 
 	testCases := []struct {
-		name        string
-		primaryCIDR string
-		PrimaryIPv6 bool
-		isDualStack bool
+		name       string
+		ipFamilies []api.IPFamily
 	}{
 		{
-			name:        "IPv4 single stack cluster",
-			primaryCIDR: "10.0.0.0/16",
-			PrimaryIPv6: false,
-			isDualStack: false,
+			name:       "IPv4 single stack cluster",
+			ipFamilies: []api.IPFamily{api.IPv4Protocol},
 		},
 		{
-			name:        "IPv6 single stack cluster",
-			primaryCIDR: "2000::/108",
-			PrimaryIPv6: true,
-			isDualStack: false,
+			name:       "IPv6 single stack cluster",
+			ipFamilies: []api.IPFamily{api.IPv6Protocol},
 		},
 
 		{
-			name:        "IPv4, IPv6 dual stack cluster",
-			primaryCIDR: "10.0.0.0/16",
-			PrimaryIPv6: false,
-			isDualStack: true,
+			name:       "IPv4, IPv6 dual stack cluster",
+			ipFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 		},
 		{
-			name:        "IPv6, IPv4 dual stack cluster",
-			primaryCIDR: "2000::/108",
-			PrimaryIPv6: true,
-			isDualStack: true,
+			name:       "IPv6, IPv4 dual stack cluster",
+			ipFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 		},
 	}
 
@@ -533,7 +547,7 @@ func TestServiceDefaulting(t *testing.T) {
 			// this func only works with dual stack feature gate on.
 			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
 
-			storage, _, server := makeStorage(t, testCase.primaryCIDR, testCase.isDualStack)
+			storage, _, server := makeStorage(t, testCase.ipFamilies)
 			defer server.Terminate(t)
 			defer storage.Store.DestroyFunc()
 
@@ -551,7 +565,7 @@ func TestServiceDefaulting(t *testing.T) {
 			defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &singleStack
 
 			// primary family
-			if testCase.PrimaryIPv6 {
+			if testCase.ipFamilies[0] == api.IPv6Protocol {
 				// no selector, gets both families
 				defaultedServiceList.Items[1].Spec.IPFamilyPolicy = &preferDualStack
 				defaultedServiceList.Items[1].Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}
@@ -559,7 +573,7 @@ func TestServiceDefaulting(t *testing.T) {
 				//assume single stack for w/selector
 				defaultedServiceList.Items[0].Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol}
 				// make dualstacked. if needed
-				if testCase.isDualStack {
+				if len(testCase.ipFamilies) > 1 {
 					defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &preferDualStack
 					defaultedServiceList.Items[0].Spec.IPFamilies = append(defaultedServiceList.Items[0].Spec.IPFamilies, api.IPv4Protocol)
 				}
@@ -571,7 +585,7 @@ func TestServiceDefaulting(t *testing.T) {
 				// assume single stack for w/selector
 				defaultedServiceList.Items[0].Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
 				// make dualstacked. if needed
-				if testCase.isDualStack {
+				if len(testCase.ipFamilies) > 1 {
 					defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &preferDualStack
 					defaultedServiceList.Items[0].Spec.IPFamilies = append(defaultedServiceList.Items[0].Spec.IPFamilies, api.IPv6Protocol)
 				}
diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go
index 10f1f985339..3439b423924 100644
--- a/pkg/registry/core/service/strategy.go
+++ b/pkg/registry/core/service/strategy.go
@@ -109,6 +109,7 @@ func (strategy svcStrategy) PrepareForCreate(ctx context.Context, obj runtime.Ob
 	service := obj.(*api.Service)
 	service.Status = api.ServiceStatus{}
 
+	//FIXME: Normalize is now called from BeginCreate in pkg/registry/core/service/storage
 	NormalizeClusterIPs(nil, service)
 	dropServiceDisabledFields(service, nil)
 }
@@ -120,6 +121,7 @@ func (strategy svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runti
 	newService.Status = oldService.Status
 
 	patchAllocatedValues(newService, oldService)
+	//FIXME: Normalize is now called from BeginUpdate in pkg/registry/core/service/storage
 	NormalizeClusterIPs(oldService, newService)
 	dropServiceDisabledFields(newService, oldService)
 	dropTypeDependentFields(newService, oldService)
@@ -362,6 +364,7 @@ func patchAllocatedValues(newSvc, oldSvc *api.Service) {
 
 // NormalizeClusterIPs adjust clusterIPs based on ClusterIP.  This must not
 // consider any other fields.
+//FIXME: move this to pkg/registry/core/service/storage
 func NormalizeClusterIPs(oldSvc, newSvc *api.Service) {
 	// In all cases here, we don't need to over-think the inputs.  Validation
 	// will be called on the new object soon enough.  All this needs to do is

From 6cc9ef3874369b64cfe3107c12a527737b1dfd20 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 00:31:07 -0800
Subject: [PATCH 09/79] Svc REST: Rename a long, hard function name

---
 pkg/registry/core/service/storage/rest.go      | 6 +++---
 pkg/registry/core/service/storage/rest_test.go | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 52e8f199020..ba429b0e087 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -185,7 +185,7 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 
 	// Ensure IP family fields are correctly initialized.  We do it here, since
 	// we want this to be visible even when dryRun == true.
-	if err := al.tryDefaultValidateServiceClusterIPFields(nil, service); err != nil {
+	if err := al.initIPFamilyFields(nil, service); err != nil {
 		return nil, err
 	}
 
@@ -395,7 +395,7 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	defer nodePortOp.Finish()
 
 	// try set ip families (for missing ip families)
-	if err := rs.alloc.tryDefaultValidateServiceClusterIPFields(oldService, service); err != nil {
+	if err := rs.alloc.initIPFamilyFields(oldService, service); err != nil {
 		return nil, false, err
 	}
 
@@ -874,7 +874,7 @@ func isMatchingPreferDualStackClusterIPFields(oldService, service *api.Service)
 
 // attempts to default service ip families according to cluster configuration
 // while ensuring that provided families are configured on cluster.
-func (al *RESTAllocStuff) tryDefaultValidateServiceClusterIPFields(oldService, service *api.Service) error {
+func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) error {
 	// can not do anything here
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 9444a651d6a..ae3ef8b195a 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -3736,7 +3736,7 @@ func TestDefaultingValidation(t *testing.T) {
 				testCase.modifyRest(storage)
 			}
 
-			err := storage.alloc.tryDefaultValidateServiceClusterIPFields(testCase.oldSvc, testCase.svc)
+			err := storage.alloc.initIPFamilyFields(testCase.oldSvc, testCase.svc)
 			if err != nil && !testCase.expectError {
 				t.Fatalf("error %v was not expected", err)
 			}

From bdbf2c6ef43f18405613f3ad379afbec761b69d3 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 08:37:10 -0800
Subject: [PATCH 10/79] Svc REST: Allow multi-IP-family in tests

---
 .../core/service/storage/storage_test.go      | 93 +++++++------------
 1 file changed, 36 insertions(+), 57 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 43b72e14350..985eccdf7c1 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -31,14 +31,14 @@ import (
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing"
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
 	netutils "k8s.io/utils/net"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	featuregatetesting "k8s.io/component-base/featuregate/testing"
-	"k8s.io/kubernetes/pkg/features"
 )
 
 func makeIPAllocator(cidr *net.IPNet) ipallocator.Interface {
@@ -49,7 +49,7 @@ func makeIPAllocator(cidr *net.IPNet) ipallocator.Interface {
 	return al
 }
 
-func newStorage(t *testing.T) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
+func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 	restOptions := generic.RESTOptions{
 		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
@@ -57,78 +57,57 @@ func newStorage(t *testing.T) (*GenericREST, *StatusREST, *etcd3testing.EtcdTest
 		DeleteCollectionWorkers: 1,
 		ResourcePrefix:          "services",
 	}
-	ipAllocs := map[api.IPFamily]ipallocator.Interface{
-		api.IPv4Protocol: makeIPAllocator(makeIPNet(t)),
+
+	ipAllocs := map[api.IPFamily]ipallocator.Interface{}
+	for _, fam := range ipFamilies {
+		switch fam {
+		case api.IPv4Protocol:
+			_, cidr, _ := netutils.ParseCIDRSloppy("10.0.0.0/16")
+			ipAllocs[fam] = makeIPAllocator(cidr)
+		case api.IPv6Protocol:
+			_, cidr, _ := netutils.ParseCIDRSloppy("2000::/108")
+			ipAllocs[fam] = makeIPAllocator(cidr)
+		default:
+			t.Fatalf("Unknown IPFamily: %v", fam)
+		}
 	}
-	serviceStorage, statusStorage, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, nil)
+
+	serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, nil)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
 	return serviceStorage, statusStorage, server
 }
 
+// This is used in generic registry tests.
 func validService() *api.Service {
-	singleStack := api.IPFamilyPolicySingleStack
-	clusterInternalTrafficPolicy := api.ServiceInternalTrafficPolicyCluster
-
-	return &api.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "foo",
-			Namespace: metav1.NamespaceDefault,
-		},
-		Spec: api.ServiceSpec{
-			Selector:        map[string]string{"bar": "baz"},
-			ClusterIP:       api.ClusterIPNone,
-			ClusterIPs:      []string{api.ClusterIPNone},
-			IPFamilyPolicy:  &singleStack,
-			IPFamilies:      []api.IPFamily{api.IPv4Protocol},
-			SessionAffinity: "None",
-			Type:            api.ServiceTypeClusterIP,
-			Ports: []api.ServicePort{{
-				Port:       6502,
-				Protocol:   api.ProtocolTCP,
-				TargetPort: intstr.FromInt(6502),
-			}},
-			InternalTrafficPolicy: &clusterInternalTrafficPolicy,
-		},
-	}
+	return svctest.MakeService("foo",
+		svctest.SetClusterIPs(api.ClusterIPNone),
+		svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+		svctest.SetIPFamilies(api.IPv4Protocol))
 }
 
 func TestCreate(t *testing.T) {
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	test := genericregistrytest.New(t, storage.Store)
-	validService := validService()
-	validService.ObjectMeta = metav1.ObjectMeta{}
+	svc := validService()
+	svc.ObjectMeta = metav1.ObjectMeta{} // because genericregistrytest
 	test.TestCreate(
 		// valid
-		validService,
+		svc,
 		// invalid
 		&api.Service{
 			Spec: api.ServiceSpec{},
 		},
-		// invalid
-		&api.Service{
-			Spec: api.ServiceSpec{
-				Selector:        map[string]string{"bar": "baz"},
-				ClusterIPs:      []string{"invalid"},
-				SessionAffinity: "None",
-				Type:            api.ServiceTypeClusterIP,
-				Ports: []api.ServicePort{{
-					Port:       6502,
-					Protocol:   api.ProtocolTCP,
-					TargetPort: intstr.FromInt(6502),
-				}},
-			},
-		},
 	)
 }
 
 func TestUpdate(t *testing.T) {
 	clusterInternalTrafficPolicy := api.ServiceInternalTrafficPolicyCluster
 
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	test := genericregistrytest.New(t, storage.Store).AllowCreateOnUpdate()
@@ -157,7 +136,7 @@ func TestUpdate(t *testing.T) {
 }
 
 func TestDelete(t *testing.T) {
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	test := genericregistrytest.New(t, storage.Store).AllowCreateOnUpdate().ReturnDeletedObject()
@@ -165,7 +144,7 @@ func TestDelete(t *testing.T) {
 }
 
 func TestGet(t *testing.T) {
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	test := genericregistrytest.New(t, storage.Store).AllowCreateOnUpdate()
@@ -173,7 +152,7 @@ func TestGet(t *testing.T) {
 }
 
 func TestList(t *testing.T) {
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	test := genericregistrytest.New(t, storage.Store).AllowCreateOnUpdate()
@@ -181,7 +160,7 @@ func TestList(t *testing.T) {
 }
 
 func TestWatch(t *testing.T) {
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	test := genericregistrytest.New(t, storage.Store)
@@ -205,7 +184,7 @@ func TestWatch(t *testing.T) {
 }
 
 func TestShortNames(t *testing.T) {
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	expected := []string{"svc"}
@@ -213,7 +192,7 @@ func TestShortNames(t *testing.T) {
 }
 
 func TestCategories(t *testing.T) {
-	storage, _, server := newStorage(t)
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 	expected := []string{"all"}

From e4c6d0837e16c84712c482dc0428c2bc46a5936e Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Thu, 8 Jul 2021 11:55:20 -0700
Subject: [PATCH 11/79] Svc REST: Rename some tests for clarity

---
 .../core/service/storage/storage_test.go         | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 985eccdf7c1..6eb40de2c94 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -87,7 +87,7 @@ func validService() *api.Service {
 		svctest.SetIPFamilies(api.IPv4Protocol))
 }
 
-func TestCreate(t *testing.T) {
+func TestGenericCreate(t *testing.T) {
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
@@ -104,7 +104,7 @@ func TestCreate(t *testing.T) {
 	)
 }
 
-func TestUpdate(t *testing.T) {
+func TestGenericUpdate(t *testing.T) {
 	clusterInternalTrafficPolicy := api.ServiceInternalTrafficPolicyCluster
 
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
@@ -135,7 +135,7 @@ func TestUpdate(t *testing.T) {
 	)
 }
 
-func TestDelete(t *testing.T) {
+func TestGenericDelete(t *testing.T) {
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
@@ -143,7 +143,7 @@ func TestDelete(t *testing.T) {
 	test.TestDelete(validService())
 }
 
-func TestGet(t *testing.T) {
+func TestGenericGet(t *testing.T) {
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
@@ -151,7 +151,7 @@ func TestGet(t *testing.T) {
 	test.TestGet(validService())
 }
 
-func TestList(t *testing.T) {
+func TestGenericList(t *testing.T) {
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
@@ -159,7 +159,7 @@ func TestList(t *testing.T) {
 	test.TestList(validService())
 }
 
-func TestWatch(t *testing.T) {
+func TestGenericWatch(t *testing.T) {
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
@@ -183,7 +183,7 @@ func TestWatch(t *testing.T) {
 	)
 }
 
-func TestShortNames(t *testing.T) {
+func TestGenericShortNames(t *testing.T) {
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
@@ -191,7 +191,7 @@ func TestShortNames(t *testing.T) {
 	registrytest.AssertShortNames(t, storage, expected)
 }
 
-func TestCategories(t *testing.T) {
+func TestGenericCategories(t *testing.T) {
 	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()

From 237434bd42d671035a11cbbae7dfd14a042fb5d8 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 08:37:52 -0800
Subject: [PATCH 12/79] Svc REST: Overhaul Create test wrt dual-stack

This includes a few cases.

1) TestCreateIgnoresIPFamilyForExternalName: Prove that ExternalName is
ignored for dual-stack.  A small set of test cases were chosen to
demonstrate.

2) TestCreateIgnoresIPFamilyWithoutDualStack: Prove that when the
dual-stack gate is off, all services are ignored for dual-stack.  A
small set of test cases were chosen to demonstrate

3) TestCreateInitIPFields: Run over a huge array of test cases for
dual-stack.  This was generated by this program:
https://gist.github.com/thockin/cccc9c9a580b4830ee0946ddd43eeafe and
then updated by hand.
---
 pkg/api/service/testing/make.go               |   23 +
 .../core/service/storage/storage_test.go      | 4561 +++++++++++++++++
 2 files changed, 4584 insertions(+)

diff --git a/pkg/api/service/testing/make.go b/pkg/api/service/testing/make.go
index 6d1473247b7..1aad4814f96 100644
--- a/pkg/api/service/testing/make.go
+++ b/pkg/api/service/testing/make.go
@@ -113,6 +113,29 @@ func MakeServicePort(name string, port int, tgtPort intstr.IntOrString, proto ap
 	}
 }
 
+// SetHeadless sets the service as headless and clears other fields.
+func SetHeadless(svc *api.Service) {
+	SetTypeClusterIP(svc)
+	svc.Spec.ClusterIP = api.ClusterIPNone
+}
+
+// SetSelector sets the service selector.
+func SetSelector(sel map[string]string) Tweak {
+	return func(svc *api.Service) {
+		svc.Spec.Selector = map[string]string{}
+		for k, v := range sel {
+			svc.Spec.Selector[k] = v
+		}
+	}
+}
+
+// SetClusterIP sets the service ClusterIP fields.
+func SetClusterIP(ip string) Tweak {
+	return func(svc *api.Service) {
+		svc.Spec.ClusterIP = ip
+	}
+}
+
 // SetClusterIPs sets the service ClusterIP and ClusterIPs fields.
 func SetClusterIPs(ips ...string) Tweak {
 	return func(svc *api.Service) {
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 6eb40de2c94..9e60205e77a 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -28,8 +28,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing"
+	"k8s.io/apiserver/pkg/registry/rest"
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -614,3 +616,4562 @@ func TestServiceDefaulting(t *testing.T) {
 		})
 	}
 }
+
+func fmtIPFamilyPolicy(pol *api.IPFamilyPolicyType) string {
+	if pol == nil {
+		return "<nil>"
+	}
+	return string(*pol)
+}
+
+func fmtIPFamilies(fams []api.IPFamily) string {
+	if fams == nil {
+		return "[]"
+	}
+	return fmt.Sprintf("%v", fams)
+}
+
+func familyOf(ip string) api.IPFamily {
+	if netutils.IsIPv4String(ip) {
+		return api.IPv4Protocol
+	}
+	if netutils.IsIPv6String(ip) {
+		return api.IPv6Protocol
+	}
+	return api.IPFamily("unknown")
+}
+
+// Prove that create ignores IPFamily stuff when type is ExternalName.
+func TestCreateIgnoresIPFamilyForExternalName(t *testing.T) {
+	type testCase struct {
+		name           string
+		svc            *api.Service
+		expectError    bool
+		expectPolicy   *api.IPFamilyPolicyType
+		expectFamilies []api.IPFamily
+	}
+	// These cases were chosen from the full gamut to ensure all "interesting"
+	// cases are covered.
+	testCases := []struct {
+		name            string
+		clusterFamilies []api.IPFamily
+		enableDualStack bool
+		cases           []testCase
+	}{{
+		name:            "singlestack:v4_gate:off",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		cases: []testCase{{
+			name:           "Policy:unset_Families:unset",
+			svc:            svctest.MakeService("foo"),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:SingleStack_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:PreferDualStack_Families:v4v6",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:RequireDualStack_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}},
+	}, {
+		name:            "singlestack:v6_gate:on",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		enableDualStack: true,
+		cases: []testCase{{
+			name:           "Policy:unset_Families:unset",
+			svc:            svctest.MakeService("foo"),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:SingleStack_Families:v6",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectError: true,
+		}, {
+			name: "Policy:PreferDualStack_Families:v4v6",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectError: true,
+		}, {
+			name: "Policy:RequireDualStack_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectError: true,
+		}},
+	}, {
+		name:            "dualstack:v4v6_gate:off",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		enableDualStack: false,
+		cases: []testCase{{
+			name:           "Policy:unset_Families:unset",
+			svc:            svctest.MakeService("foo"),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:SingleStack_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:PreferDualStack_Families:v4v6",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:RequireDualStack_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}},
+	}, {
+		name:            "dualstack:v6v4_gate:on",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+		enableDualStack: true,
+		cases: []testCase{{
+			name:           "Policy:unset_Families:unset",
+			svc:            svctest.MakeService("foo"),
+			expectPolicy:   nil,
+			expectFamilies: nil,
+		}, {
+			name: "Policy:SingleStack_Families:v6",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectError: true,
+		}, {
+			name: "Policy:PreferDualStack_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectError: true,
+		}, {
+			name: "Policy:RequireDualStack_Families:v4v6",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectError: true,
+		}},
+	}}
+
+	for _, otc := range testCases {
+		t.Run(otc.name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, otc.enableDualStack)()
+
+			storage, _, server := newStorage(t, otc.clusterFamilies)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			for _, itc := range otc.cases {
+				t.Run(itc.name, func(t *testing.T) {
+					// This test is ONLY ExternalName services.
+					itc.svc.Spec.Type = api.ServiceTypeExternalName
+					itc.svc.Spec.ExternalName = "example.com"
+
+					ctx := genericapirequest.NewDefaultContext()
+					createdObj, err := storage.Create(ctx, itc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+					if itc.expectError && err != nil {
+						return
+					}
+					if err != nil {
+						t.Fatalf("unexpected error creating service: %v", err)
+					}
+					defer storage.Delete(ctx, itc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+					if itc.expectError && err == nil {
+						t.Fatalf("unexpected success creating service")
+					}
+					createdSvc := createdObj.(*api.Service)
+
+					if want, got := fmtIPFamilyPolicy(itc.expectPolicy), fmtIPFamilyPolicy(createdSvc.Spec.IPFamilyPolicy); want != got {
+						t.Errorf("wrong IPFamilyPolicy: want %s, got %s", want, got)
+					}
+					if want, got := fmtIPFamilies(itc.expectFamilies), fmtIPFamilies(createdSvc.Spec.IPFamilies); want != got {
+						t.Errorf("wrong IPFamilies: want %s, got %s", want, got)
+					}
+				})
+			}
+		})
+	}
+}
+
+// Prove that create ignores IPFamily stuff when dual-stack is disabled.
+func TestCreateIgnoresIPFamilyWithoutDualStack(t *testing.T) {
+	// These cases were chosen from the full gamut to ensure all "interesting"
+	// cases are covered.
+	testCases := []struct {
+		name string
+		svc  *api.Service
+	}{
+		//----------------------------------------
+		// ClusterIP:unset
+		//----------------------------------------
+		{
+			name: "ClusterIP:unset_Policy:unset_Families:unset",
+			svc:  svctest.MakeService("foo"),
+		}, {
+			name: "ClusterIP:unset_Policy:unset_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+		}, {
+			name: "ClusterIP:unset_Policy:unset_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+		}, {
+			name: "ClusterIP:unset_Policy:SingleStack_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+		}, {
+			name: "ClusterIP:unset_Policy:SingleStack_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+		}, {
+			name: "ClusterIP:unset_Policy:SingleStack_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+		}, {
+			name: "ClusterIP:unset_Policy:PreferDualStack_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+		}, {
+			name: "ClusterIP:unset_Policy:PreferDualStack_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+		}, {
+			name: "ClusterIP:unset_Policy:PreferDualStack_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+		}, {
+			name: "ClusterIP:unset_Policy:RequireDualStack_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+		}, {
+			name: "ClusterIP:unset_Policy:RequireDualStack_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+		}, {
+			name: "ClusterIP:unset_Policy:RequireDualStack_Families:v6v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+		},
+		//----------------------------------------
+		// ClusterIPs:v4v6
+		//----------------------------------------
+		{
+			name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+		}, {
+			name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+		}, {
+			name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+		},
+		//----------------------------------------
+		// Headless
+		//----------------------------------------
+		{
+			name: "Headless_Policy:unset_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless),
+		}, {
+			name: "Headless_Policy:unset_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+		}, {
+			name: "Headless_Policy:RequireDualStack_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+		},
+		//----------------------------------------
+		// HeadlessSelectorless
+		//----------------------------------------
+		{
+			name: "HeadlessSelectorless_Policy:unset_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetSelector(nil)),
+		}, {
+			name: "HeadlessSelectorless_Policy:unset_Families:v4",
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetSelector(nil),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+		}, {
+			name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetSelector(nil),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+		},
+	}
+
+	// This test is ONLY with the gate off.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, false)()
+
+	// Do this in the outer scope for performance.
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			defer storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			createdSvc := createdObj.(*api.Service)
+			//FIXME: HACK!!  Delete above calls "inner" which doesn't
+			// yet call the allocators - no release = alloc errors!
+			defer func() {
+				for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
+					for _, ip := range createdSvc.Spec.ClusterIPs {
+						al.Release(netutils.ParseIPSloppy(ip))
+					}
+				}
+			}()
+
+			// The gate is off - these should always be empty.
+			if want, got := fmtIPFamilyPolicy(nil), fmtIPFamilyPolicy(createdSvc.Spec.IPFamilyPolicy); want != got {
+				t.Errorf("wrong IPFamilyPolicy: want %s, got %s", want, got)
+			}
+			if want, got := fmtIPFamilies(nil), fmtIPFamilies(createdSvc.Spec.IPFamilies); want != got {
+				t.Errorf("wrong IPFamilies: want %s, got %s", want, got)
+			}
+		})
+	}
+}
+
+// Prove that create initializes clusterIPs from clusterIP.  This simplifies
+// later tests to not need to re-prove this.
+func TestCreateInitClusterIPsFromClusterIP(t *testing.T) {
+	testCases := []struct {
+		name            string
+		clusterFamilies []api.IPFamily
+		svc             *api.Service
+	}{{
+		name:            "singlestack:v4_clusterip:unset",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v4_clusterip:set",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIP("10.0.0.1")),
+	}, {
+		name:            "singlestack:v6_clusterip:unset",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v6_clusterip:set",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIP("2000::1")),
+	}, {
+		name:            "dualstack:v4v6_clusterip:unset",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "dualstack:v4v6_clusterip:set",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIP("10.0.0.1")),
+	}, {
+		name:            "dualstack:v6v4_clusterip:unset",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "dualstack:v6v4_clusterip:set",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIP("2000::1")),
+	}}
+
+	// This test is ONLY with the gate enabled.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			storage, _, server := newStorage(t, tc.clusterFamilies)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			createdSvc := createdObj.(*api.Service)
+
+			if createdSvc.Spec.ClusterIP == "" {
+				t.Errorf("expected ClusterIP to be set")
+
+			}
+			if tc.svc.Spec.ClusterIP != "" {
+				if want, got := tc.svc.Spec.ClusterIP, createdSvc.Spec.ClusterIP; want != got {
+					t.Errorf("wrong ClusterIP: want %s, got %s", want, got)
+				}
+			}
+			if len(createdSvc.Spec.ClusterIPs) == 0 {
+				t.Errorf("expected ClusterIPs to be set")
+			}
+			if want, got := createdSvc.Spec.ClusterIP, createdSvc.Spec.ClusterIPs[0]; want != got {
+				t.Errorf("wrong ClusterIPs[0]: want %s, got %s", want, got)
+			}
+		})
+	}
+}
+
+// Prove that create initializes IPFamily fields correctly.
+func TestCreateInitIPFields(t *testing.T) {
+	type testCase struct {
+		name           string
+		svc            *api.Service
+		expectError    bool
+		expectPolicy   api.IPFamilyPolicyType
+		expectFamilies []api.IPFamily
+		expectHeadless bool
+	}
+	// These cases were chosen from the full gamut to ensure all "interesting"
+	// cases are covered.
+	testCases := []struct {
+		name            string
+		clusterFamilies []api.IPFamily
+		cases           []testCase
+	}{
+		{
+			name:            "singlestack:v4",
+			clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+			cases: []testCase{
+				//----------------------------------------
+				// singlestack:v4 ClusterIPs:unset
+				//----------------------------------------
+				{
+					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					svc:            svctest.MakeService("foo"),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v4 ClusterIPs:v4
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v4_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1")),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v4 ClusterIPs:v4v6
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v4 ClusterIPs:v6v4
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v4 Headless
+				//----------------------------------------
+				{
+					name: "Headless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v4 HeadlessSelectorless
+				//----------------------------------------
+				{
+					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				},
+			},
+		}, {
+			name:            "singlestack:v6",
+			clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+			cases: []testCase{
+				//----------------------------------------
+				// singlestack:v6 ClusterIPs:unset
+				//----------------------------------------
+				{
+					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					svc:            svctest.MakeService("foo"),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v6 ClusterIPs:v6
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v6_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1")),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v6 ClusterIPs:v4v6
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v6 ClusterIPs:v6v4
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v6 Headless
+				//----------------------------------------
+				{
+					name: "Headless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// singlestack:v6 HeadlessSelectorless
+				//----------------------------------------
+				{
+					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				},
+			},
+		}, {
+			name:            "dualstack:v4v6",
+			clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+			cases: []testCase{
+				//----------------------------------------
+				// dualstack:v4v6 ClusterIPs:unset
+				//----------------------------------------
+				{
+					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					svc:            svctest.MakeService("foo"),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				},
+				//----------------------------------------
+				// dualstack:v4v6 ClusterIPs:v4
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v4_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1")),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// dualstack:v4v6 ClusterIPs:v6
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v6_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1")),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				},
+				//----------------------------------------
+				// dualstack:v4v6 ClusterIPs:v4v6
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// dualstack:v4v6 ClusterIPs:v6v4
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				},
+				//----------------------------------------
+				// dualstack:v4v6 Headless
+				//----------------------------------------
+				{
+					name: "Headless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				},
+				//----------------------------------------
+				// dualstack:v4v6 HeadlessSelectorless
+				//----------------------------------------
+				{
+					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				},
+			},
+		}, {
+			name:            "dualstack:v6v4",
+			clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+			cases: []testCase{
+				//----------------------------------------
+				// dualstack:v6v4 ClusterIPs:unset
+				//----------------------------------------
+				{
+					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					svc:            svctest.MakeService("foo"),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				},
+				//----------------------------------------
+				// dualstack:v6v4 ClusterIPs:v4
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v4_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1")),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// dualstack:v6v4 ClusterIPs:v6
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v6_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1")),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				},
+				//----------------------------------------
+				// dualstack:v6v4 ClusterIPs:v4v6
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+				}, {
+					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				},
+				//----------------------------------------
+				// dualstack:v6v4 ClusterIPs:v6v4
+				//----------------------------------------
+				{
+					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+				},
+				//----------------------------------------
+				// dualstack:v6v4 Headless
+				//----------------------------------------
+				{
+					name: "Headless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				},
+				//----------------------------------------
+				// dualstack:v6v4 HeadlessSelectorless
+				//----------------------------------------
+				{
+					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicySingleStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectError: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectHeadless: true,
+				}, {
+					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					svc: svctest.MakeService("foo",
+						svctest.SetHeadless,
+						svctest.SetSelector(nil),
+						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
+					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectHeadless: true,
+				},
+			},
+		},
+	}
+
+	// This test is ONLY with the gate enabled.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	for _, otc := range testCases {
+		t.Run(otc.name, func(t *testing.T) {
+
+			// Do this in the outer loop for performance.
+			storage, _, server := newStorage(t, otc.clusterFamilies)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			for _, itc := range otc.cases {
+				t.Run(itc.name, func(t *testing.T) {
+					ctx := genericapirequest.NewDefaultContext()
+					createdObj, err := storage.Create(ctx, itc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+					if itc.expectError && err != nil {
+						return
+					}
+					if err != nil {
+						t.Fatalf("unexpected error creating service: %v", err)
+					}
+					defer storage.Delete(ctx, itc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+					if itc.expectError && err == nil {
+						t.Fatalf("unexpected success creating service")
+					}
+					createdSvc := createdObj.(*api.Service)
+					//FIXME: HACK!!  Delete above calls "inner" which doesn't
+					// yet call the allocators - no release = alloc errors!
+					defer func() {
+						for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
+							for _, ip := range createdSvc.Spec.ClusterIPs {
+								al.Release(netutils.ParseIPSloppy(ip))
+							}
+						}
+					}()
+
+					if want, got := fmtIPFamilyPolicy(&itc.expectPolicy), fmtIPFamilyPolicy(createdSvc.Spec.IPFamilyPolicy); want != got {
+						t.Errorf("wrong IPFamilyPolicy: want %s, got %s", want, got)
+					}
+					if want, got := fmtIPFamilies(itc.expectFamilies), fmtIPFamilies(createdSvc.Spec.IPFamilies); want != got {
+						t.Errorf("wrong IPFamilies: want %s, got %s", want, got)
+					}
+					if itc.expectHeadless {
+						if !reflect.DeepEqual(createdSvc.Spec.ClusterIPs, []string{"None"}) {
+							t.Errorf("wrong clusterIPs: want [\"None\"], got %v", createdSvc.Spec.ClusterIPs)
+						}
+					} else {
+						if c, f := len(createdSvc.Spec.ClusterIPs), len(createdSvc.Spec.IPFamilies); c != f {
+							t.Errorf("clusterIPs and ipFamilies are not the same length: %d vs %d", c, f)
+						}
+						for i, clip := range createdSvc.Spec.ClusterIPs {
+							if cf, ef := familyOf(clip), createdSvc.Spec.IPFamilies[i]; cf != ef {
+								t.Errorf("clusterIP is the wrong IP family: want %s, got %s", ef, cf)
+							}
+						}
+					}
+				})
+			}
+		})
+	}
+}
+
+// func TestServiceRegistryCreateDryRun(t *testing.T) {
+// 	requireDualStack := api.IPFamilyPolicyRequireDualStack
+// 	testCases := []struct {
+// 		name            string
+// 		svc             *api.Service
+// 		enableDualStack bool
+// 	}{
+// 		{
+// 			name:            "v4 service featuregate off",
+// 			enableDualStack: false,
+// 			svc: &api.Service{
+// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+// 				Spec: api.ServiceSpec{
+// 					Selector:        map[string]string{"bar": "baz"},
+// 					SessionAffinity: api.ServiceAffinityNone,
+// 					Type:            api.ServiceTypeClusterIP,
+// 					ClusterIP:       "1.2.3.4",
+// 					ClusterIPs:      []string{"1.2.3.4"},
+// 					Ports: []api.ServicePort{{
+// 						Port:       6502,
+// 						Protocol:   api.ProtocolTCP,
+// 						TargetPort: intstr.FromInt(6502),
+// 					}},
+// 				},
+// 			},
+// 		},
+// 		{
+// 			name:            "v6 service featuregate on but singlestack",
+// 			enableDualStack: true,
+// 			svc: &api.Service{
+// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+// 				Spec: api.ServiceSpec{
+// 					Selector:        map[string]string{"bar": "baz"},
+// 					SessionAffinity: api.ServiceAffinityNone,
+// 					Type:            api.ServiceTypeClusterIP,
+// 					IPFamilies:      []api.IPFamily{api.IPv6Protocol},
+// 					ClusterIP:       "2000:0:0:0:0:0:0:1",
+// 					ClusterIPs:      []string{"2000:0:0:0:0:0:0:1"},
+// 					Ports: []api.ServicePort{{
+// 						Port:       6502,
+// 						Protocol:   api.ProtocolTCP,
+// 						TargetPort: intstr.FromInt(6502),
+// 					}},
+// 				},
+// 			},
+// 		},
+// 		{
+// 			name:            "dualstack v4,v6 service",
+// 			enableDualStack: true,
+// 			svc: &api.Service{
+// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+// 				Spec: api.ServiceSpec{
+// 					Selector:        map[string]string{"bar": "baz"},
+// 					SessionAffinity: api.ServiceAffinityNone,
+// 					Type:            api.ServiceTypeClusterIP,
+// 					IPFamilyPolicy:  &requireDualStack,
+// 					ClusterIP:       "1.2.3.4",
+// 					ClusterIPs:      []string{"1.2.3.4", "2000:0:0:0:0:0:0:1"},
+// 					IPFamilies:      []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+// 					Ports: []api.ServicePort{{
+// 						Port:       6502,
+// 						Protocol:   api.ProtocolTCP,
+// 						TargetPort: intstr.FromInt(6502),
+// 					}},
+// 				},
+// 			},
+// 		},
+// 		{
+// 			name:            "dualstack v6,v4 service",
+// 			enableDualStack: true,
+// 			svc: &api.Service{
+// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+// 				Spec: api.ServiceSpec{
+// 					Selector:        map[string]string{"bar": "baz"},
+// 					SessionAffinity: api.ServiceAffinityNone,
+// 					Type:            api.ServiceTypeClusterIP,
+// 					IPFamilyPolicy:  &requireDualStack,
+// 					ClusterIP:       "2000:0:0:0:0:0:0:1",
+// 					ClusterIPs:      []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"},
+// 					IPFamilies:      []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+// 					Ports: []api.ServicePort{{
+// 						Port:       6502,
+// 						Protocol:   api.ProtocolTCP,
+// 						TargetPort: intstr.FromInt(6502),
+// 					}},
+// 				},
+// 			},
+// 		},
+// 	}
+//
+// 	for _, tc := range testCases {
+// 		t.Run(tc.name, func(t *testing.T) {
+// 			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
+//
+// 			families := []api.IPFamily{api.IPv4Protocol}
+// 			if tc.enableDualStack {
+// 				families = append(families, api.IPv6Protocol)
+// 			}
+// 			storage, registry, server := NewTestREST(t, nil, families)
+// 			defer server.Terminate(t)
+//
+// 			ctx := genericapirequest.NewDefaultContext()
+// 			_, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
+// 			if err != nil {
+// 				t.Fatalf("Unexpected error: %v", err)
+// 			}
+//
+// 			for i, family := range tc.svc.Spec.IPFamilies {
+// 				alloc := storage.alloc.serviceIPAllocatorsByFamily[family]
+// 				if alloc.Has(netutils.ParseIPSloppy(tc.svc.Spec.ClusterIPs[i])) {
+// 					t.Errorf("unexpected side effect: ip allocated %v", tc.svc.Spec.ClusterIPs[i])
+// 				}
+// 			}
+//
+// 			srv, err := registry.GetService(ctx, tc.svc.Name, &metav1.GetOptions{})
+// 			if err != nil {
+// 				t.Errorf("unexpected error: %v", err)
+// 			}
+// 			if srv != nil {
+// 				t.Errorf("unexpected service found: %v", srv)
+// 			}
+// 		})
+// 	}
+// }

From 52856f3fbed42d463746e8c2d0249b3da2bfa300 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 13:07:36 -0800
Subject: [PATCH 13/79] Add dry-run support to the IP allocator subsystem

---
 .../core/service/ipallocator/allocator.go     | 88 +++++++++++++++++-
 .../service/ipallocator/allocator_test.go     | 92 ++++++++++++++++---
 2 files changed, 161 insertions(+), 19 deletions(-)

diff --git a/pkg/registry/core/service/ipallocator/allocator.go b/pkg/registry/core/service/ipallocator/allocator.go
index df8a2841ac0..820ae4a73e7 100644
--- a/pkg/registry/core/service/ipallocator/allocator.go
+++ b/pkg/registry/core/service/ipallocator/allocator.go
@@ -37,6 +37,9 @@ type Interface interface {
 	CIDR() net.IPNet
 	IPFamily() api.IPFamily
 	Has(ip net.IP) bool
+
+	// DryRun offers a way to try operations without persisting them.
+	DryRun() Interface
 }
 
 var (
@@ -46,11 +49,12 @@ var (
 )
 
 type ErrNotInRange struct {
+	IP         net.IP
 	ValidRange string
 }
 
 func (e *ErrNotInRange) Error() string {
-	return fmt.Sprintf("provided IP is not in the valid range. The range of valid IPs is %s", e.ValidRange)
+	return fmt.Sprintf("the provided IP (%v) is not in the valid range. The range of valid IPs is %s", e.IP, e.ValidRange)
 }
 
 // Range is a contiguous block of IPs that can be allocated atomically.
@@ -98,11 +102,13 @@ func New(cidr *net.IPNet, allocatorFactory allocator.AllocatorFactory) (*Range,
 		}
 	} else {
 		family = api.IPv4Protocol
-		// Don't use the IPv4 network's broadcast address.
+		// Don't use the IPv4 network's broadcast address, but don't just
+		// Allocate() it - we don't ever want to be able to release it.
 		max--
 	}
 
-	// Don't use the network's ".0" address.
+	// Don't use the network's ".0" address, but don't just Allocate() it - we
+	// don't ever want to be able to release it.
 	base.Add(base, big.NewInt(1))
 	max--
 
@@ -114,6 +120,7 @@ func New(cidr *net.IPNet, allocatorFactory allocator.AllocatorFactory) (*Range,
 	}
 	var err error
 	r.alloc, err = allocatorFactory(r.max, rangeSpec)
+
 	return &r, err
 }
 
@@ -162,18 +169,35 @@ func (r *Range) CIDR() net.IPNet {
 	return *r.net
 }
 
+// DryRun returns a non-persisting form of this Range.
+func (r *Range) DryRun() Interface {
+	return dryRunRange{r}
+}
+
+// For clearer code.
+const dryRunTrue = true
+const dryRunFalse = false
+
 // Allocate attempts to reserve the provided IP. ErrNotInRange or
 // ErrAllocated will be returned if the IP is not valid for this range
 // or has already been reserved.  ErrFull will be returned if there
 // are no addresses left.
 func (r *Range) Allocate(ip net.IP) error {
+	return r.allocate(ip, dryRunFalse)
+}
+
+func (r *Range) allocate(ip net.IP, dryRun bool) error {
 	label := r.CIDR()
 	ok, offset := r.contains(ip)
 	if !ok {
 		// update metrics
 		clusterIPAllocationErrors.WithLabelValues(label.String()).Inc()
-
-		return &ErrNotInRange{r.net.String()}
+		return &ErrNotInRange{ip, r.net.String()}
+	}
+	if dryRun {
+		// Don't bother to check whether the IP is actually free. It's racy and
+		// not worth the effort to plumb any further.
+		return nil
 	}
 
 	allocated, err := r.alloc.Allocate(offset)
@@ -200,7 +224,17 @@ func (r *Range) Allocate(ip net.IP) error {
 // AllocateNext reserves one of the IPs from the pool. ErrFull may
 // be returned if there are no addresses left.
 func (r *Range) AllocateNext() (net.IP, error) {
+	return r.allocateNext(dryRunFalse)
+}
+
+func (r *Range) allocateNext(dryRun bool) (net.IP, error) {
 	label := r.CIDR()
+	if dryRun {
+		// Don't bother finding a free value. It's racy and not worth the
+		// effort to plumb any further.
+		return r.CIDR().IP, nil
+	}
+
 	offset, ok, err := r.alloc.AllocateNext()
 	if err != nil {
 		// update metrics
@@ -226,10 +260,17 @@ func (r *Range) AllocateNext() (net.IP, error) {
 // unallocated IP or an IP out of the range is a no-op and
 // returns no error.
 func (r *Range) Release(ip net.IP) error {
+	return r.release(ip, dryRunFalse)
+}
+
+func (r *Range) release(ip net.IP, dryRun bool) error {
 	ok, offset := r.contains(ip)
 	if !ok {
 		return nil
 	}
+	if dryRun {
+		return nil
+	}
 
 	err := r.alloc.Release(offset)
 	if err == nil {
@@ -312,3 +353,40 @@ func (r *Range) contains(ip net.IP) (bool, int) {
 func calculateIPOffset(base *big.Int, ip net.IP) int {
 	return int(big.NewInt(0).Sub(netutils.BigForIP(ip), base).Int64())
 }
+
+// dryRunRange is a shim to satisfy Interface without persisting state.
+type dryRunRange struct {
+	real *Range
+}
+
+func (dry dryRunRange) Allocate(ip net.IP) error {
+	return dry.real.allocate(ip, dryRunTrue)
+}
+
+func (dry dryRunRange) AllocateNext() (net.IP, error) {
+	return dry.real.allocateNext(dryRunTrue)
+}
+
+func (dry dryRunRange) Release(ip net.IP) error {
+	return dry.real.release(ip, dryRunTrue)
+}
+
+func (dry dryRunRange) ForEach(cb func(net.IP)) {
+	dry.real.ForEach(cb)
+}
+
+func (dry dryRunRange) CIDR() net.IPNet {
+	return dry.real.CIDR()
+}
+
+func (dry dryRunRange) IPFamily() api.IPFamily {
+	return dry.real.IPFamily()
+}
+
+func (dry dryRunRange) DryRun() Interface {
+	return dry
+}
+
+func (dry dryRunRange) Has(ip net.IP) bool {
+	return dry.real.Has(ip)
+}
diff --git a/pkg/registry/core/service/ipallocator/allocator_test.go b/pkg/registry/core/service/ipallocator/allocator_test.go
index 8580303adb4..bf4dec2fa83 100644
--- a/pkg/registry/core/service/ipallocator/allocator_test.go
+++ b/pkg/registry/core/service/ipallocator/allocator_test.go
@@ -76,34 +76,34 @@ func TestAllocate(t *testing.T) {
 		}
 		t.Logf("base: %v", r.base.Bytes())
 		if f := r.Free(); f != tc.free {
-			t.Errorf("Test %s unexpected free %d", tc.name, f)
+			t.Errorf("[%s] wrong free: expected %d, got %d", tc.name, tc.free, f)
 		}
 
 		rCIDR := r.CIDR()
 		if rCIDR.String() != tc.cidr {
-			t.Errorf("allocator returned a different cidr")
+			t.Errorf("[%s] wrong CIDR: expected %v, got %v", tc.name, tc.cidr, rCIDR.String())
 		}
 
 		if r.IPFamily() != tc.family {
-			t.Errorf("allocator returned wrong IP family")
+			t.Errorf("[%s] wrong IP family: expected %v, got %v", tc.name, tc.family, r.IPFamily())
 		}
 
 		if f := r.Used(); f != 0 {
-			t.Errorf("Test %s unexpected used %d", tc.name, f)
+			t.Errorf("[%s]: wrong used: expected %d, got %d", tc.name, 0, f)
 		}
 		found := sets.NewString()
 		count := 0
 		for r.Free() > 0 {
 			ip, err := r.AllocateNext()
 			if err != nil {
-				t.Fatalf("Test %s error @ %d: %v", tc.name, count, err)
+				t.Fatalf("[%s] error @ %d: %v", tc.name, count, err)
 			}
 			count++
 			if !cidr.Contains(ip) {
-				t.Fatalf("Test %s allocated %s which is outside of %s", tc.name, ip, cidr)
+				t.Fatalf("[%s] allocated %s which is outside of %s", tc.name, ip, cidr)
 			}
 			if found.Has(ip.String()) {
-				t.Fatalf("Test %s allocated %s twice @ %d", tc.name, ip, count)
+				t.Fatalf("[%s] allocated %s twice @ %d", tc.name, ip, count)
 			}
 			found.Insert(ip.String())
 		}
@@ -116,17 +116,17 @@ func TestAllocate(t *testing.T) {
 			t.Fatal(err)
 		}
 		if f := r.Free(); f != 1 {
-			t.Errorf("Test %s unexpected free %d", tc.name, f)
+			t.Errorf("[%s] wrong free: expected %d, got %d", tc.name, 1, f)
 		}
 		if f := r.Used(); f != (tc.free - 1) {
-			t.Errorf("Test %s unexpected free %d", tc.name, f)
+			t.Errorf("[%s] wrong free: expected %d, got %d", tc.name, tc.free-1, f)
 		}
 		ip, err := r.AllocateNext()
 		if err != nil {
 			t.Fatal(err)
 		}
 		if !released.Equal(ip) {
-			t.Errorf("Test %s unexpected %s : %s", tc.name, ip, released)
+			t.Errorf("[%s] unexpected %s : %s", tc.name, ip, released)
 		}
 
 		if err := r.Release(released); err != nil {
@@ -142,19 +142,19 @@ func TestAllocate(t *testing.T) {
 			t.Fatal(err)
 		}
 		if f := r.Free(); f != 1 {
-			t.Errorf("Test %s unexpected free %d", tc.name, f)
+			t.Errorf("[%s] wrong free: expected %d, got %d", tc.name, 1, f)
 		}
 		if f := r.Used(); f != (tc.free - 1) {
-			t.Errorf("Test %s unexpected free %d", tc.name, f)
+			t.Errorf("[%s] wrong free: expected %d, got %d", tc.name, tc.free-1, f)
 		}
 		if err := r.Allocate(released); err != nil {
 			t.Fatal(err)
 		}
 		if f := r.Free(); f != 0 {
-			t.Errorf("Test %s unexpected free %d", tc.name, f)
+			t.Errorf("[%s] wrong free: expected %d, got %d", tc.name, 0, f)
 		}
 		if f := r.Used(); f != tc.free {
-			t.Errorf("Test %s unexpected free %d", tc.name, f)
+			t.Errorf("[%s] wrong free: expected %d, got %d", tc.name, tc.free, f)
 		}
 	}
 }
@@ -514,3 +514,67 @@ func expectMetrics(t *testing.T, label string, em testMetrics) {
 		t.Fatalf("metrics error: expected %v, received %v", em, m)
 	}
 }
+
+func TestDryRun(t *testing.T) {
+	testCases := []struct {
+		name   string
+		cidr   string
+		family api.IPFamily
+	}{{
+		name:   "IPv4",
+		cidr:   "192.168.1.0/24",
+		family: api.IPv4Protocol,
+	}, {
+		name:   "IPv6",
+		cidr:   "2001:db8:1::/48",
+		family: api.IPv6Protocol,
+	}}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, cidr, err := netutils.ParseCIDRSloppy(tc.cidr)
+			if err != nil {
+				t.Fatalf("unexpected failure: %v", err)
+			}
+			r, err := NewInMemory(cidr)
+			if err != nil {
+				t.Fatalf("unexpected failure: %v", err)
+			}
+
+			baseUsed := r.Used()
+
+			rCIDR := r.DryRun().CIDR()
+			if rCIDR.String() != tc.cidr {
+				t.Errorf("allocator returned a different cidr")
+			}
+
+			if r.DryRun().IPFamily() != tc.family {
+				t.Errorf("allocator returned wrong IP family")
+			}
+
+			expectUsed := func(t *testing.T, r *Range, expect int) {
+				t.Helper()
+				if u := r.Used(); u != expect {
+					t.Errorf("unexpected used count: got %d, wanted %d", u, expect)
+				}
+			}
+			expectUsed(t, r, baseUsed)
+
+			err = r.DryRun().Allocate(netutils.AddIPOffset(netutils.BigForIP(cidr.IP), 1))
+			if err != nil {
+				t.Fatalf("unexpected failure: %v", err)
+			}
+			expectUsed(t, r, baseUsed)
+
+			_, err = r.DryRun().AllocateNext()
+			if err != nil {
+				t.Fatalf("unexpected failure: %v", err)
+			}
+			expectUsed(t, r, baseUsed)
+
+			if err := r.DryRun().Release(cidr.IP); err != nil {
+				t.Fatalf("unexpected failure: %v", err)
+			}
+			expectUsed(t, r, baseUsed)
+		})
+	}
+}

From e338c9db4b441b48dbbedc03ab5716d8da9bdc39 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 15:28:35 -0800
Subject: [PATCH 14/79] Svc REST: Set Cluster IPs during dry-run Create

Dry-run should behave like a real API call and return valid results.
---
 pkg/api/service/testing/make.go               |   7 +
 pkg/registry/core/service/storage/rest.go     |  46 ++--
 .../core/service/storage/rest_test.go         |   4 +-
 .../core/service/storage/storage_test.go      | 238 +++++++++---------
 4 files changed, 149 insertions(+), 146 deletions(-)

diff --git a/pkg/api/service/testing/make.go b/pkg/api/service/testing/make.go
index 1aad4814f96..6be0abe6c4a 100644
--- a/pkg/api/service/testing/make.go
+++ b/pkg/api/service/testing/make.go
@@ -192,6 +192,13 @@ func SetAllocateLoadBalancerNodePorts(val bool) Tweak {
 	}
 }
 
+// SetUniqueNodePorts sets all nodeports to unique values.
+func SetUniqueNodePorts(svc *api.Service) {
+	for i := range svc.Spec.Ports {
+		svc.Spec.Ports[i].NodePort = int32(30000 + i)
+	}
+}
+
 // SetHealthCheckNodePort sets the healthCheckNodePort field for a Service.
 func SetHealthCheckNodePort(value int32) Tweak {
 	return func(svc *api.Service) {
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index ba429b0e087..1f3d07dd805 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -190,15 +190,14 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 	}
 
 	// Allocate ClusterIPs
-	//FIXME: we need to put values in, even if dry run - else validation should
-	//not pass.  It does but that should be fixed.
-	if !dryRun {
-		if txn, err := al.allocServiceClusterIPsNew(service); err != nil {
-			result.Revert()
-			return nil, err
-		} else {
-			result = append(result, txn)
-		}
+	//TODO(thockin): validation should not pass with empty clusterIP, but it
+	//does (and is tested!).  Fixing that all is a big PR and will have to
+	//happen later.
+	if txn, err := al.allocServiceClusterIPsNew(service, dryRun); err != nil {
+		result.Revert()
+		return nil, err
+	} else {
+		result = append(result, txn)
 	}
 
 	// Allocate ports
@@ -378,6 +377,7 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	// on failure: any ip that should be released, will *not* be released
 	// on success: any ip that should be released, will  be released
 	defer func() {
+		//FIXME: plumb dryRun down here
 		// release the allocated, this is expected to be cleared if the entire function ran to success
 		if allocated_released, err := rs.alloc.releaseClusterIPs(allocated); err != nil {
 			klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v", service.Namespace, service.Name, err, allocated, allocated_released)
@@ -385,6 +385,7 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 		}
 		// performRelease is set when the enture function ran to success
 		if performRelease {
+			//FIXME: plumb dryRun down here
 			if toReleaseIPs_released, err := rs.alloc.releaseClusterIPs(toReleaseIPs); err != nil {
 				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v", service.Namespace, service.Name, err, toReleaseIPs, toReleaseIPs_released)
 			}
@@ -527,11 +528,14 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO
 	return r.services.ConvertToTable(ctx, object, tableOptions)
 }
 
-func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]string) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
 	allocated := make(map[api.IPFamily]string)
 
 	for family, ip := range toAlloc {
 		allocator := al.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
+		if dryRun {
+			allocator = allocator.DryRun()
+		}
 		if ip == "" {
 			allocatedIP, err := allocator.AllocateNext()
 			if err != nil {
@@ -577,13 +581,13 @@ func (al *RESTAllocStuff) releaseClusterIPs(toRelease map[api.IPFamily]string) (
 
 // standard allocator for dualstackgate==Off, hard wired dependency
 // and ignores policy, families and clusterIPs
-func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
 	toAlloc := make(map[api.IPFamily]string)
 
 	// get clusterIP.. empty string if user did not specify an ip
 	toAlloc[al.defaultServiceIPFamily] = service.Spec.ClusterIP
 	// alloc
-	allocated, err := al.allocClusterIPs(service, toAlloc)
+	allocated, err := al.allocClusterIPs(service, toAlloc, dryRun)
 
 	// set
 	if err == nil {
@@ -595,16 +599,19 @@ func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service) (map[api.I
 }
 
 //FIXME: merge into allocServiceClusterIPs rather than call it
-func (al *RESTAllocStuff) allocServiceClusterIPsNew(service *api.Service) (transaction, error) {
+func (al *RESTAllocStuff) allocServiceClusterIPsNew(service *api.Service, dryRun bool) (transaction, error) {
 	// clusterIPs that were allocated may need to be released in case of
 	// failure at a higher level.
-	toReleaseClusterIPs, err := al.allocServiceClusterIPs(service)
+	toReleaseClusterIPs, err := al.allocServiceClusterIPs(service, dryRun)
 	if err != nil {
 		return nil, err
 	}
 
 	txn := callbackTransaction{
 		revert: func() {
+			if dryRun {
+				return
+			}
 			released, err := al.releaseClusterIPs(toReleaseClusterIPs)
 			if err != nil {
 				klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v",
@@ -616,7 +623,7 @@ func (al *RESTAllocStuff) allocServiceClusterIPsNew(service *api.Service) (trans
 }
 
 // allocates ClusterIPs for a service
-func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
 	// external name don't get ClusterIPs
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil, nil
@@ -628,7 +635,7 @@ func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service) (map[api.
 	}
 
 	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return al.allocServiceClusterIP(service)
+		return al.allocServiceClusterIP(service, dryRun)
 	}
 
 	toAlloc := make(map[api.IPFamily]string)
@@ -651,7 +658,7 @@ func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service) (map[api.
 	}
 
 	// allocate
-	allocated, err := al.allocClusterIPs(service, toAlloc)
+	allocated, err := al.allocClusterIPs(service, toAlloc, dryRun)
 
 	// set if successful
 	if err == nil {
@@ -702,7 +709,8 @@ func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Serv
 	// CASE A:
 	// Update service from ExternalName to non-ExternalName, should initialize ClusterIP.
 	if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName {
-		allocated, err := al.allocServiceClusterIPs(service)
+		//FIXME: plumb dryRun down here
+		allocated, err := al.allocServiceClusterIPs(service, false)
 		return allocated, nil, err
 	}
 
@@ -748,7 +756,7 @@ func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Serv
 		toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1]
 
 		// allocate
-		allocated, err := al.allocClusterIPs(service, toAllocate)
+		allocated, err := al.allocClusterIPs(service, toAllocate, false) //FIXME: plumb dry-run down here
 		// set if successful
 		if err == nil {
 			service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]]
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index ae3ef8b195a..d00aa81a5bf 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -1062,7 +1062,7 @@ func TestAllocateLoadBalancerNodePorts(t *testing.T) {
 		name: "allocate false, gate on, port specified",
 		svc: svctest.MakeService("alloc-false-specific",
 			svctest.SetTypeLoadBalancer,
-			svctest.SetNodePorts(30000),
+			svctest.SetUniqueNodePorts,
 			svctest.SetAllocateLoadBalancerNodePorts(false)),
 		expectNodePorts:      true,
 		allocateNodePortGate: true,
@@ -1070,7 +1070,7 @@ func TestAllocateLoadBalancerNodePorts(t *testing.T) {
 		name: "allocate true, gate on, port specified",
 		svc: svctest.MakeService("alloc-true-specific",
 			svctest.SetTypeLoadBalancer,
-			svctest.SetNodePorts(30000),
+			svctest.SetUniqueNodePorts,
 			svctest.SetAllocateLoadBalancerNodePorts(true)),
 		expectNodePorts:      true,
 		allocateNodePortGate: true,
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 9e60205e77a..a8fc1f47e1a 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	machineryutilnet "k8s.io/apimachinery/pkg/util/net"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing"
@@ -39,6 +40,7 @@ import (
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
+	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
 	netutils "k8s.io/utils/net"
 )
@@ -51,6 +53,14 @@ func makeIPAllocator(cidr *net.IPNet) ipallocator.Interface {
 	return al
 }
 
+func makePortAllocator(ports machineryutilnet.PortRange) portallocator.Interface {
+	al, err := portallocator.NewInMemory(ports)
+	if err != nil {
+		panic(fmt.Sprintf("error creating port allocator: %v", err))
+	}
+	return al
+}
+
 func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 	restOptions := generic.RESTOptions{
@@ -74,7 +84,9 @@ func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusR
 		}
 	}
 
-	serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, nil)
+	portAlloc := makePortAllocator(*(machineryutilnet.ParsePortRangeOrDie("30000-32767")))
+
+	serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, portAlloc)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
@@ -5051,127 +5063,103 @@ func TestCreateInitIPFields(t *testing.T) {
 	}
 }
 
-// func TestServiceRegistryCreateDryRun(t *testing.T) {
-// 	requireDualStack := api.IPFamilyPolicyRequireDualStack
-// 	testCases := []struct {
-// 		name            string
-// 		svc             *api.Service
-// 		enableDualStack bool
-// 	}{
-// 		{
-// 			name:            "v4 service featuregate off",
-// 			enableDualStack: false,
-// 			svc: &api.Service{
-// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
-// 				Spec: api.ServiceSpec{
-// 					Selector:        map[string]string{"bar": "baz"},
-// 					SessionAffinity: api.ServiceAffinityNone,
-// 					Type:            api.ServiceTypeClusterIP,
-// 					ClusterIP:       "1.2.3.4",
-// 					ClusterIPs:      []string{"1.2.3.4"},
-// 					Ports: []api.ServicePort{{
-// 						Port:       6502,
-// 						Protocol:   api.ProtocolTCP,
-// 						TargetPort: intstr.FromInt(6502),
-// 					}},
-// 				},
-// 			},
-// 		},
-// 		{
-// 			name:            "v6 service featuregate on but singlestack",
-// 			enableDualStack: true,
-// 			svc: &api.Service{
-// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
-// 				Spec: api.ServiceSpec{
-// 					Selector:        map[string]string{"bar": "baz"},
-// 					SessionAffinity: api.ServiceAffinityNone,
-// 					Type:            api.ServiceTypeClusterIP,
-// 					IPFamilies:      []api.IPFamily{api.IPv6Protocol},
-// 					ClusterIP:       "2000:0:0:0:0:0:0:1",
-// 					ClusterIPs:      []string{"2000:0:0:0:0:0:0:1"},
-// 					Ports: []api.ServicePort{{
-// 						Port:       6502,
-// 						Protocol:   api.ProtocolTCP,
-// 						TargetPort: intstr.FromInt(6502),
-// 					}},
-// 				},
-// 			},
-// 		},
-// 		{
-// 			name:            "dualstack v4,v6 service",
-// 			enableDualStack: true,
-// 			svc: &api.Service{
-// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
-// 				Spec: api.ServiceSpec{
-// 					Selector:        map[string]string{"bar": "baz"},
-// 					SessionAffinity: api.ServiceAffinityNone,
-// 					Type:            api.ServiceTypeClusterIP,
-// 					IPFamilyPolicy:  &requireDualStack,
-// 					ClusterIP:       "1.2.3.4",
-// 					ClusterIPs:      []string{"1.2.3.4", "2000:0:0:0:0:0:0:1"},
-// 					IPFamilies:      []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-// 					Ports: []api.ServicePort{{
-// 						Port:       6502,
-// 						Protocol:   api.ProtocolTCP,
-// 						TargetPort: intstr.FromInt(6502),
-// 					}},
-// 				},
-// 			},
-// 		},
-// 		{
-// 			name:            "dualstack v6,v4 service",
-// 			enableDualStack: true,
-// 			svc: &api.Service{
-// 				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
-// 				Spec: api.ServiceSpec{
-// 					Selector:        map[string]string{"bar": "baz"},
-// 					SessionAffinity: api.ServiceAffinityNone,
-// 					Type:            api.ServiceTypeClusterIP,
-// 					IPFamilyPolicy:  &requireDualStack,
-// 					ClusterIP:       "2000:0:0:0:0:0:0:1",
-// 					ClusterIPs:      []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"},
-// 					IPFamilies:      []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-// 					Ports: []api.ServicePort{{
-// 						Port:       6502,
-// 						Protocol:   api.ProtocolTCP,
-// 						TargetPort: intstr.FromInt(6502),
-// 					}},
-// 				},
-// 			},
-// 		},
-// 	}
-//
-// 	for _, tc := range testCases {
-// 		t.Run(tc.name, func(t *testing.T) {
-// 			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
-//
-// 			families := []api.IPFamily{api.IPv4Protocol}
-// 			if tc.enableDualStack {
-// 				families = append(families, api.IPv6Protocol)
-// 			}
-// 			storage, registry, server := NewTestREST(t, nil, families)
-// 			defer server.Terminate(t)
-//
-// 			ctx := genericapirequest.NewDefaultContext()
-// 			_, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
-// 			if err != nil {
-// 				t.Fatalf("Unexpected error: %v", err)
-// 			}
-//
-// 			for i, family := range tc.svc.Spec.IPFamilies {
-// 				alloc := storage.alloc.serviceIPAllocatorsByFamily[family]
-// 				if alloc.Has(netutils.ParseIPSloppy(tc.svc.Spec.ClusterIPs[i])) {
-// 					t.Errorf("unexpected side effect: ip allocated %v", tc.svc.Spec.ClusterIPs[i])
-// 				}
-// 			}
-//
-// 			srv, err := registry.GetService(ctx, tc.svc.Name, &metav1.GetOptions{})
-// 			if err != nil {
-// 				t.Errorf("unexpected error: %v", err)
-// 			}
-// 			if srv != nil {
-// 				t.Errorf("unexpected service found: %v", srv)
-// 			}
-// 		})
-// 	}
-// }
+// Prove that a dry-run create doesn't actually allocate IPs or ports.
+func TestCreateDryRun(t *testing.T) {
+	testCases := []struct {
+		name            string
+		clusterFamilies []api.IPFamily
+		enableDualStack bool
+		svc             *api.Service
+	}{{
+		name:            "singlestack:v4_gate:off_clusterip:unset",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v4_gate:off_clusterip:set",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo", svctest.SetClusterIPs("10.0.0.1")),
+	}, {
+		name:            "singlestack:v6_gate:on_clusterip:unset",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v6_gate:on_clusterip:set",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo", svctest.SetClusterIPs("2000::1")),
+	}, {
+		name:            "dualstack:v4v6_gate:on_clusterip:unset",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo", svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+	}, {
+		name:            "dualstack:v4v6_gate:on_clusterip:set",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo", svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack), svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+	}, {
+		name:            "singlestack:v4_gate:off_type:NodePort_nodeport:unset",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo", svctest.SetTypeNodePort),
+	}, {
+		name:            "singlestack:v4_gate:on_type:LoadBalancer_nodePort:set",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo", svctest.SetTypeLoadBalancer, svctest.SetUniqueNodePorts),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
+
+			storage, _, server := newStorage(t, tc.clusterFamilies)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			createdSvc := createdObj.(*api.Service)
+
+			// Ensure IPs were allocated
+			if netutils.ParseIPSloppy(createdSvc.Spec.ClusterIP) == nil {
+				t.Errorf("expected valid clusterIP: %v", createdSvc.Spec.ClusterIP)
+			}
+
+			// Ensure the IP allocators are clean.
+			if !tc.enableDualStack {
+				if storage.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol].Has(netutils.ParseIPSloppy(createdSvc.Spec.ClusterIP)) {
+					t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIP)
+				}
+			} else {
+				for _, ip := range createdSvc.Spec.ClusterIPs {
+					if netutils.ParseIPSloppy(ip) == nil {
+						t.Errorf("expected valid clusterIP: %v", createdSvc.Spec.ClusterIP)
+					}
+				}
+				for i, fam := range createdSvc.Spec.IPFamilies {
+					if storage.alloc.serviceIPAllocatorsByFamily[fam].Has(netutils.ParseIPSloppy(createdSvc.Spec.ClusterIPs[i])) {
+						t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIPs[i])
+					}
+				}
+			}
+
+			if tc.svc.Spec.Type != api.ServiceTypeClusterIP {
+				for _, p := range createdSvc.Spec.Ports {
+					if p.NodePort == 0 {
+						t.Errorf("expected a NodePort value")
+					}
+					if storage.alloc.serviceNodePorts.Has(int(p.NodePort)) {
+						t.Errorf("expected port to not be allocated: %v", p.NodePort)
+					}
+				}
+			}
+		})
+	}
+}

From b880d3a149e86ee43bb0a69b9d189ef125ac110c Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 2 Jul 2021 19:40:37 -0700
Subject: [PATCH 15/79] Svc REST: better test checks in new tests

"Has()" was strengthened in the older rest_test, now in the newer.
---
 .../core/service/storage/rest_test.go         | 19 -------------
 .../core/service/storage/storage_test.go      | 28 +++++++++++++++----
 2 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index d00aa81a5bf..edcc8157bcb 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -288,25 +288,6 @@ func makeIPNet6(t *testing.T) *net.IPNet {
 	return net
 }
 
-func ipIsAllocated(t *testing.T, alloc ipallocator.Interface, ipstr string) bool {
-	t.Helper()
-	ip := netutils.ParseIPSloppy(ipstr)
-	if ip == nil {
-		t.Errorf("error parsing IP %q", ipstr)
-		return false
-	}
-	return alloc.Has(ip)
-}
-
-func portIsAllocated(t *testing.T, alloc portallocator.Interface, port int32) bool {
-	t.Helper()
-	if port == 0 {
-		t.Errorf("port is 0")
-		return false
-	}
-	return alloc.Has(int(port))
-}
-
 func TestServiceRegistryCreate(t *testing.T) {
 	testCases := []struct {
 		svc             *api.Service
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index a8fc1f47e1a..6397beb9e39 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -61,6 +61,25 @@ func makePortAllocator(ports machineryutilnet.PortRange) portallocator.Interface
 	return al
 }
 
+func ipIsAllocated(t *testing.T, alloc ipallocator.Interface, ipstr string) bool {
+	t.Helper()
+	ip := netutils.ParseIPSloppy(ipstr)
+	if ip == nil {
+		t.Errorf("error parsing IP %q", ipstr)
+		return false
+	}
+	return alloc.Has(ip)
+}
+
+func portIsAllocated(t *testing.T, alloc portallocator.Interface, port int32) bool {
+	t.Helper()
+	if port == 0 {
+		t.Errorf("port is 0")
+		return false
+	}
+	return alloc.Has(int(port))
+}
+
 func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 	restOptions := generic.RESTOptions{
@@ -5134,7 +5153,7 @@ func TestCreateDryRun(t *testing.T) {
 
 			// Ensure the IP allocators are clean.
 			if !tc.enableDualStack {
-				if storage.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol].Has(netutils.ParseIPSloppy(createdSvc.Spec.ClusterIP)) {
+				if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol], createdSvc.Spec.ClusterIP) {
 					t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIP)
 				}
 			} else {
@@ -5144,7 +5163,7 @@ func TestCreateDryRun(t *testing.T) {
 					}
 				}
 				for i, fam := range createdSvc.Spec.IPFamilies {
-					if storage.alloc.serviceIPAllocatorsByFamily[fam].Has(netutils.ParseIPSloppy(createdSvc.Spec.ClusterIPs[i])) {
+					if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
 						t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIPs[i])
 					}
 				}
@@ -5152,10 +5171,7 @@ func TestCreateDryRun(t *testing.T) {
 
 			if tc.svc.Spec.Type != api.ServiceTypeClusterIP {
 				for _, p := range createdSvc.Spec.Ports {
-					if p.NodePort == 0 {
-						t.Errorf("expected a NodePort value")
-					}
-					if storage.alloc.serviceNodePorts.Has(int(p.NodePort)) {
+					if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
 						t.Errorf("expected port to not be allocated: %v", p.NodePort)
 					}
 				}

From ca4a95ee49e27363ad611600e82eb45f55a4d26d Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 09:34:00 -0800
Subject: [PATCH 16/79] Svc REST: Dedup tests for defaulting

---
 .../core/service/storage/storage_test.go      | 377 +-----------------
 1 file changed, 22 insertions(+), 355 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 6397beb9e39..3730b1bc4d4 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -232,159 +232,12 @@ func TestGenericCategories(t *testing.T) {
 	registrytest.AssertCategories(t, storage, expected)
 }
 
-func makeServiceList() (undefaulted, defaulted *api.ServiceList) {
-	undefaulted = &api.ServiceList{Items: []api.Service{}}
-	defaulted = &api.ServiceList{Items: []api.Service{}}
-
-	singleStack := api.IPFamilyPolicySingleStack
-	requireDualStack := api.IPFamilyPolicyRequireDualStack
-
-	var undefaultedSvc *api.Service
-	var defaultedSvc *api.Service
-
-	// (for headless) tests must set fields  manually according to how the cluster configured
-	// headless w selector (subject to how the cluster is configured)
-	undefaultedSvc = &api.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: "headless_with_selector", ResourceVersion: "1", Namespace: metav1.NamespaceDefault},
-		Spec: api.ServiceSpec{
-			Type:       api.ServiceTypeClusterIP,
-			ClusterIPs: []string{api.ClusterIPNone},
-			Selector:   map[string]string{"foo": "bar"},
-		},
-	}
-	defaultedSvc = undefaultedSvc.DeepCopy()
-	defaultedSvc.Spec.IPFamilyPolicy = nil // forcing tests to set them
-	defaultedSvc.Spec.IPFamilies = nil     // forcing tests to them
-
-	undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc))
-	defaulted.Items = append(defaulted.Items, *(defaultedSvc))
-
-	// headless w/o selector (always set to require and families according to cluster)
-	undefaultedSvc = &api.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: "headless_no_selector", ResourceVersion: "1", Namespace: metav1.NamespaceDefault},
-		Spec: api.ServiceSpec{
-			Type:       api.ServiceTypeClusterIP,
-			ClusterIPs: []string{api.ClusterIPNone},
-			Selector:   nil,
-		},
-	}
-	defaultedSvc = undefaultedSvc.DeepCopy()
-	defaultedSvc.Spec.IPFamilyPolicy = nil // forcing tests to set them
-	defaultedSvc.Spec.IPFamilies = nil     // forcing tests to them
-
-	undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc))
-	defaulted.Items = append(defaulted.Items, *(defaultedSvc))
-
-	// single stack IPv4
-	undefaultedSvc = &api.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: "ipv4", ResourceVersion: "1", Namespace: metav1.NamespaceDefault},
-		Spec: api.ServiceSpec{
-			Type:      api.ServiceTypeClusterIP,
-			ClusterIP: "10.0.0.4",
-		},
-	}
-	defaultedSvc = undefaultedSvc.DeepCopy()
-	defaultedSvc.Spec.IPFamilyPolicy = &singleStack
-	defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
-
-	undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc))
-	defaulted.Items = append(defaulted.Items, *(defaultedSvc))
-
-	// single stack IPv6
-	undefaultedSvc = &api.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: "ipv6", ResourceVersion: "1", Namespace: metav1.NamespaceDefault},
-		Spec: api.ServiceSpec{
-			Type:      api.ServiceTypeClusterIP,
-			ClusterIP: "2000::1",
-		},
-	}
-	defaultedSvc = undefaultedSvc.DeepCopy()
-	defaultedSvc.Spec.IPFamilyPolicy = &singleStack
-	defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol}
-
-	undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc))
-	defaulted.Items = append(defaulted.Items, *(defaultedSvc))
-
-	// dualstack IPv4 IPv6
-	undefaultedSvc = &api.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: "ipv4_ipv6", ResourceVersion: "1", Namespace: metav1.NamespaceDefault},
-		Spec: api.ServiceSpec{
-			Type:       api.ServiceTypeClusterIP,
-			ClusterIP:  "10.0.0.4",
-			ClusterIPs: []string{"10.0.0.4", "2000::1"},
-		},
-	}
-	defaultedSvc = undefaultedSvc.DeepCopy()
-	defaultedSvc.Spec.IPFamilyPolicy = &requireDualStack
-	defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-
-	undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc))
-	defaulted.Items = append(defaulted.Items, *(defaultedSvc))
-
-	// dualstack IPv6 IPv4
-	undefaultedSvc = &api.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: "ipv6_ipv4", ResourceVersion: "1", Namespace: metav1.NamespaceDefault},
-		Spec: api.ServiceSpec{
-			Type:       api.ServiceTypeClusterIP,
-			ClusterIP:  "2000::1",
-			ClusterIPs: []string{"2000::1", "10.0.0.4"},
-		},
-	}
-	defaultedSvc = undefaultedSvc.DeepCopy()
-	defaultedSvc.Spec.IPFamilyPolicy = &requireDualStack
-	defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}
-
-	undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc))
-	defaulted.Items = append(defaulted.Items, *(defaultedSvc))
-
-	// external name
-	undefaultedSvc = &api.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: "external_name", ResourceVersion: "1", Namespace: metav1.NamespaceDefault},
-		Spec: api.ServiceSpec{
-			Type: api.ServiceTypeExternalName,
-		},
-	}
-
-	defaultedSvc = undefaultedSvc.DeepCopy()
-	defaultedSvc.Spec.IPFamilyPolicy = nil
-	defaultedSvc.Spec.IPFamilies = nil
-
-	undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc))
-	defaulted.Items = append(defaulted.Items, *(defaultedSvc))
-
-	return undefaulted, defaulted
-}
-
 func TestServiceDefaultOnRead(t *testing.T) {
-	// Helper makes a mostly-valid Service.  Test-cases can tweak it as needed.
-	makeService := func(tweak func(*api.Service)) *api.Service {
-		svc := &api.Service{
-			ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "ns"},
-			Spec: api.ServiceSpec{
-				Type:       api.ServiceTypeClusterIP,
-				ClusterIP:  "1.2.3.4",
-				ClusterIPs: []string{"1.2.3.4"},
-			},
-		}
-		if tweak != nil {
-			tweak(svc)
-		}
-		return svc
-	}
 	// Helper makes a mostly-valid ServiceList.  Test-cases can tweak it as needed.
-	makeServiceList := func(tweak func(*api.ServiceList)) *api.ServiceList {
+	makeServiceList := func(tweaks ...svctest.Tweak) *api.ServiceList {
+		svc := svctest.MakeService("foo", tweaks...)
 		list := &api.ServiceList{
-			Items: []api.Service{{
-				ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "ns"},
-				Spec: api.ServiceSpec{
-					Type:       api.ServiceTypeClusterIP,
-					ClusterIP:  "1.2.3.4",
-					ClusterIPs: []string{"1.2.3.4"},
-				},
-			}},
-		}
-		if tweak != nil {
-			tweak(list)
+			Items: []api.Service{*svc},
 		}
 		return list
 	}
@@ -395,75 +248,38 @@ func TestServiceDefaultOnRead(t *testing.T) {
 		expect runtime.Object
 	}{{
 		name:   "no change v4",
-		input:  makeService(nil),
-		expect: makeService(nil),
+		input:  svctest.MakeService("foo", svctest.SetClusterIPs("10.0.0.1")),
+		expect: svctest.MakeService("foo", svctest.SetClusterIPs("10.0.0.1")),
 	}, {
-		name: "missing clusterIPs v4",
-		input: makeService(func(svc *api.Service) {
-			svc.Spec.ClusterIPs = nil
-		}),
-		expect: makeService(nil),
+		name:   "missing clusterIPs v4",
+		input:  svctest.MakeService("foo", svctest.SetClusterIP("10.0.0.1")),
+		expect: svctest.MakeService("foo", svctest.SetClusterIPs("10.0.0.1")),
 	}, {
-		name: "no change v6",
-		input: makeService(func(svc *api.Service) {
-			svc.Spec.ClusterIP = "2000::"
-			svc.Spec.ClusterIPs = []string{"2000::"}
-		}),
-		expect: makeService(func(svc *api.Service) {
-			svc.Spec.ClusterIP = "2000::"
-			svc.Spec.ClusterIPs = []string{"2000::"}
-		}),
+		name:   "no change v6",
+		input:  svctest.MakeService("foo", svctest.SetClusterIPs("2000::1")),
+		expect: svctest.MakeService("foo", svctest.SetClusterIPs("2000::1")),
 	}, {
-		name: "missing clusterIPs v6",
-		input: makeService(func(svc *api.Service) {
-			svc.Spec.ClusterIP = "2000::"
-			svc.Spec.ClusterIPs = nil
-		}),
-		expect: makeService(func(svc *api.Service) {
-			svc.Spec.ClusterIP = "2000::"
-			svc.Spec.ClusterIPs = []string{"2000::"}
-		}),
+		name:   "missing clusterIPs v6",
+		input:  svctest.MakeService("foo", svctest.SetClusterIP("2000::1")),
+		expect: svctest.MakeService("foo", svctest.SetClusterIPs("2000::1")),
 	}, {
 		name:   "list, no change v4",
-		input:  makeServiceList(nil),
-		expect: makeServiceList(nil),
+		input:  makeServiceList(svctest.SetClusterIPs("10.0.0.1")),
+		expect: makeServiceList(svctest.SetClusterIPs("10.0.0.1")),
 	}, {
-		name: "list, missing clusterIPs v4",
-		input: makeServiceList(func(list *api.ServiceList) {
-			list.Items[0].Spec.ClusterIPs = nil
-		}),
-		expect: makeService(nil),
+		name:   "list, missing clusterIPs v4",
+		input:  makeServiceList(svctest.SetClusterIP("10.0.0.1")),
+		expect: makeServiceList(svctest.SetClusterIPs("10.0.0.1")),
 	}, {
 		name:  "not Service or ServiceList",
 		input: &api.Pod{},
 	}}
 
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
 	for _, tc := range testCases {
-		makeStorage := func(t *testing.T) (*GenericREST, *etcd3testing.EtcdTestServer) {
-			etcdStorage, server := registrytest.NewEtcdStorage(t, "")
-			restOptions := generic.RESTOptions{
-				StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
-				Decorator:               generic.UndecoratedStorage,
-				DeleteCollectionWorkers: 1,
-				ResourcePrefix:          "services",
-			}
-
-			_, cidr, err := netutils.ParseCIDRSloppy("10.0.0.0/24")
-			if err != nil {
-				t.Fatalf("failed to parse CIDR")
-			}
-
-			ipAllocs := map[api.IPFamily]ipallocator.Interface{
-				api.IPv4Protocol: makeIPAllocator(cidr),
-			}
-			serviceStorage, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, nil)
-			if err != nil {
-				t.Fatalf("unexpected error from REST storage: %v", err)
-			}
-			return serviceStorage, server
-		}
 		t.Run(tc.name, func(t *testing.T) {
-			storage, server := makeStorage(t)
+			storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
 			defer server.Terminate(t)
 			defer storage.Store.DestroyFunc()
 
@@ -499,155 +315,6 @@ func TestServiceDefaultOnRead(t *testing.T) {
 	}
 }
 
-func TestServiceDefaulting(t *testing.T) {
-	makeStorage := func(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
-		etcdStorage, server := registrytest.NewEtcdStorage(t, "")
-		restOptions := generic.RESTOptions{
-			StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
-			Decorator:               generic.UndecoratedStorage,
-			DeleteCollectionWorkers: 1,
-			ResourcePrefix:          "services",
-		}
-
-		ipAllocs := map[api.IPFamily]ipallocator.Interface{}
-		for _, fam := range ipFamilies {
-			switch fam {
-			case api.IPv4Protocol:
-				_, cidr, _ := netutils.ParseCIDRSloppy("10.0.0.0/16")
-				ipAllocs[fam] = makeIPAllocator(cidr)
-			case api.IPv6Protocol:
-				_, cidr, _ := netutils.ParseCIDRSloppy("2000::/108")
-				ipAllocs[fam] = makeIPAllocator(cidr)
-			}
-		}
-
-		serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, nil)
-		if err != nil {
-			t.Fatalf("unexpected error from REST storage: %v", err)
-		}
-		return serviceStorage, statusStorage, server
-	}
-
-	testCases := []struct {
-		name       string
-		ipFamilies []api.IPFamily
-	}{
-		{
-			name:       "IPv4 single stack cluster",
-			ipFamilies: []api.IPFamily{api.IPv4Protocol},
-		},
-		{
-			name:       "IPv6 single stack cluster",
-			ipFamilies: []api.IPFamily{api.IPv6Protocol},
-		},
-
-		{
-			name:       "IPv4, IPv6 dual stack cluster",
-			ipFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-		},
-		{
-			name:       "IPv6, IPv4 dual stack cluster",
-			ipFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-		},
-	}
-
-	singleStack := api.IPFamilyPolicySingleStack
-	preferDualStack := api.IPFamilyPolicyPreferDualStack
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			// this func only works with dual stack feature gate on.
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-
-			storage, _, server := makeStorage(t, testCase.ipFamilies)
-			defer server.Terminate(t)
-			defer storage.Store.DestroyFunc()
-
-			undefaultedServiceList, defaultedServiceList := makeServiceList()
-			// set the two special ones (0: w/ selector, 1: w/o selector)
-			// review default*OnRead(...)
-			// Single stack cluster:
-			// headless w/selector => singlestack
-			// headless w/o selector => preferDualStack
-			// dual stack cluster:
-			// headless w/selector => preferDualStack
-			// headless w/o selector => preferDualStack
-
-			// assume single stack
-			defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &singleStack
-
-			// primary family
-			if testCase.ipFamilies[0] == api.IPv6Protocol {
-				// no selector, gets both families
-				defaultedServiceList.Items[1].Spec.IPFamilyPolicy = &preferDualStack
-				defaultedServiceList.Items[1].Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}
-
-				//assume single stack for w/selector
-				defaultedServiceList.Items[0].Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol}
-				// make dualstacked. if needed
-				if len(testCase.ipFamilies) > 1 {
-					defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &preferDualStack
-					defaultedServiceList.Items[0].Spec.IPFamilies = append(defaultedServiceList.Items[0].Spec.IPFamilies, api.IPv4Protocol)
-				}
-			} else {
-				// no selector gets both families
-				defaultedServiceList.Items[1].Spec.IPFamilyPolicy = &preferDualStack
-				defaultedServiceList.Items[1].Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-
-				// assume single stack for w/selector
-				defaultedServiceList.Items[0].Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
-				// make dualstacked. if needed
-				if len(testCase.ipFamilies) > 1 {
-					defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &preferDualStack
-					defaultedServiceList.Items[0].Spec.IPFamilies = append(defaultedServiceList.Items[0].Spec.IPFamilies, api.IPv6Protocol)
-				}
-			}
-
-			// data is now ready for testing over various cluster configuration
-			compareSvc := func(out api.Service, expected api.Service) {
-				if expected.Spec.IPFamilyPolicy == nil && out.Spec.IPFamilyPolicy != nil {
-					t.Fatalf("service %+v expected IPFamilyPolicy to be nil", out)
-				}
-				if expected.Spec.IPFamilyPolicy != nil && out.Spec.IPFamilyPolicy == nil {
-					t.Fatalf("service %+v expected IPFamilyPolicy not to be nil", out)
-				}
-
-				if expected.Spec.IPFamilyPolicy != nil {
-					if *out.Spec.IPFamilyPolicy != *expected.Spec.IPFamilyPolicy {
-						t.Fatalf("service %+v expected IPFamilyPolicy %v got %v", out, *expected.Spec.IPFamilyPolicy, *out.Spec.IPFamilyPolicy)
-					}
-				}
-
-				if len(out.Spec.IPFamilies) != len(expected.Spec.IPFamilies) {
-					t.Fatalf("service %+v expected len(IPFamilies) == %v", out, len(expected.Spec.IPFamilies))
-				}
-				for i, ipfamily := range out.Spec.IPFamilies {
-					if expected.Spec.IPFamilies[i] != ipfamily {
-						t.Fatalf("service %+v expected ip families %+v", out, expected.Spec.IPFamilies)
-					}
-				}
-			}
-
-			copyUndefaultedList := undefaultedServiceList.DeepCopy()
-			// run for each Service
-			for i, svc := range copyUndefaultedList.Items {
-				storage.defaultOnRead(&svc)
-				compareSvc(svc, defaultedServiceList.Items[i])
-			}
-
-			copyUndefaultedList = undefaultedServiceList.DeepCopy()
-			// run as a ServiceList
-			storage.defaultOnRead(copyUndefaultedList)
-			for i, svc := range copyUndefaultedList.Items {
-				compareSvc(svc, defaultedServiceList.Items[i])
-			}
-
-			// if there are more tests needed then the last call need to work
-			// with copy of undefaulted list since
-		})
-	}
-}
-
 func fmtIPFamilyPolicy(pol *api.IPFamilyPolicyType) string {
 	if pol == nil {
 		return "<nil>"

From 9ca582f3b79001f36347f0316673a7f52dfbb251 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 16:00:55 -0800
Subject: [PATCH 17/79] Svc REST: Test that ExternalName doesn't set IPs

---
 .../core/service/storage/storage_test.go      | 58 +++++++------------
 1 file changed, 21 insertions(+), 37 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 3730b1bc4d4..4f86b8ca8ba 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -339,14 +339,12 @@ func familyOf(ip string) api.IPFamily {
 	return api.IPFamily("unknown")
 }
 
-// Prove that create ignores IPFamily stuff when type is ExternalName.
-func TestCreateIgnoresIPFamilyForExternalName(t *testing.T) {
+// Prove that create ignores IP and IPFamily stuff when type is ExternalName.
+func TestCreateIgnoresIPsForExternalName(t *testing.T) {
 	type testCase struct {
-		name           string
-		svc            *api.Service
-		expectError    bool
-		expectPolicy   *api.IPFamilyPolicyType
-		expectFamilies []api.IPFamily
+		name        string
+		svc         *api.Service
+		expectError bool
 	}
 	// These cases were chosen from the full gamut to ensure all "interesting"
 	// cases are covered.
@@ -360,41 +358,31 @@ func TestCreateIgnoresIPFamilyForExternalName(t *testing.T) {
 		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
 		enableDualStack: false,
 		cases: []testCase{{
-			name:           "Policy:unset_Families:unset",
-			svc:            svctest.MakeService("foo"),
-			expectPolicy:   nil,
-			expectFamilies: nil,
+			name: "Policy:unset_Families:unset",
+			svc:  svctest.MakeService("foo"),
 		}, {
 			name: "Policy:SingleStack_Families:v4",
 			svc: svctest.MakeService("foo",
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectPolicy:   nil,
-			expectFamilies: nil,
 		}, {
 			name: "Policy:PreferDualStack_Families:v4v6",
 			svc: svctest.MakeService("foo",
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-			expectPolicy:   nil,
-			expectFamilies: nil,
 		}, {
 			name: "Policy:RequireDualStack_Families:v6v4",
 			svc: svctest.MakeService("foo",
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-			expectPolicy:   nil,
-			expectFamilies: nil,
 		}},
 	}, {
 		name:            "singlestack:v6_gate:on",
 		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
 		enableDualStack: true,
 		cases: []testCase{{
-			name:           "Policy:unset_Families:unset",
-			svc:            svctest.MakeService("foo"),
-			expectPolicy:   nil,
-			expectFamilies: nil,
+			name: "Policy:unset_Families:unset",
+			svc:  svctest.MakeService("foo"),
 		}, {
 			name: "Policy:SingleStack_Families:v6",
 			svc: svctest.MakeService("foo",
@@ -419,41 +407,31 @@ func TestCreateIgnoresIPFamilyForExternalName(t *testing.T) {
 		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 		enableDualStack: false,
 		cases: []testCase{{
-			name:           "Policy:unset_Families:unset",
-			svc:            svctest.MakeService("foo"),
-			expectPolicy:   nil,
-			expectFamilies: nil,
+			name: "Policy:unset_Families:unset",
+			svc:  svctest.MakeService("foo"),
 		}, {
 			name: "Policy:SingleStack_Families:v4",
 			svc: svctest.MakeService("foo",
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectPolicy:   nil,
-			expectFamilies: nil,
 		}, {
 			name: "Policy:PreferDualStack_Families:v4v6",
 			svc: svctest.MakeService("foo",
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-			expectPolicy:   nil,
-			expectFamilies: nil,
 		}, {
 			name: "Policy:RequireDualStack_Families:v6v4",
 			svc: svctest.MakeService("foo",
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-			expectPolicy:   nil,
-			expectFamilies: nil,
 		}},
 	}, {
 		name:            "dualstack:v6v4_gate:on",
 		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 		enableDualStack: true,
 		cases: []testCase{{
-			name:           "Policy:unset_Families:unset",
-			svc:            svctest.MakeService("foo"),
-			expectPolicy:   nil,
-			expectFamilies: nil,
+			name: "Policy:unset_Families:unset",
+			svc:  svctest.MakeService("foo"),
 		}, {
 			name: "Policy:SingleStack_Families:v6",
 			svc: svctest.MakeService("foo",
@@ -503,12 +481,18 @@ func TestCreateIgnoresIPFamilyForExternalName(t *testing.T) {
 					}
 					createdSvc := createdObj.(*api.Service)
 
-					if want, got := fmtIPFamilyPolicy(itc.expectPolicy), fmtIPFamilyPolicy(createdSvc.Spec.IPFamilyPolicy); want != got {
+					if want, got := fmtIPFamilyPolicy(nil), fmtIPFamilyPolicy(createdSvc.Spec.IPFamilyPolicy); want != got {
 						t.Errorf("wrong IPFamilyPolicy: want %s, got %s", want, got)
 					}
-					if want, got := fmtIPFamilies(itc.expectFamilies), fmtIPFamilies(createdSvc.Spec.IPFamilies); want != got {
+					if want, got := fmtIPFamilies(nil), fmtIPFamilies(createdSvc.Spec.IPFamilies); want != got {
 						t.Errorf("wrong IPFamilies: want %s, got %s", want, got)
 					}
+					if len(createdSvc.Spec.ClusterIP) != 0 {
+						t.Errorf("expected no clusterIP, got %q", createdSvc.Spec.ClusterIP)
+					}
+					if len(createdSvc.Spec.ClusterIPs) != 0 {
+						t.Errorf("expected no clusterIPs, got %q", createdSvc.Spec.ClusterIPs)
+					}
 				})
 			}
 		})

From 0dc509a0c884d12f63918307659d2a57b7c9561a Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Nov 2020 16:18:52 -0800
Subject: [PATCH 18/79] Svc REST: Test that Headless doesn't set IPs

---
 .../core/service/storage/storage_test.go      | 76 +++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 4f86b8ca8ba..cf1ac619cb2 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -4830,3 +4830,79 @@ func TestCreateDryRun(t *testing.T) {
 		})
 	}
 }
+
+// Prove that create skips allocations for Headless services.
+func TestCreateSkipsAllocationsForHeadless(t *testing.T) {
+	testCases := []struct {
+		name            string
+		clusterFamilies []api.IPFamily
+		enableDualStack bool
+		svc             *api.Service
+		expectError     bool
+	}{{
+		name:            "singlestack:v4_gate:off_type:ClusterIP",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v6_gate:on_type:ClusterIP",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "dualstack:v4v6_gate:off_type:ClusterIP",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "dualstack:v6v4_gate:on_type:ClusterIP",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v4_gate:off_type:NodePort",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo", svctest.SetTypeNodePort),
+		expectError:     true,
+	}, {
+		name:            "singlestack:v6_gate:on_type:LoadBalancer",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+		expectError:     true,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
+
+			storage, _, server := newStorage(t, tc.clusterFamilies)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			// This test is ONLY headless services.
+			tc.svc.Spec.ClusterIP = api.ClusterIPNone
+
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if tc.expectError && err != nil {
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			if tc.expectError && err == nil {
+				t.Fatalf("unexpected success creating service")
+			}
+			createdSvc := createdObj.(*api.Service)
+
+			if createdSvc.Spec.ClusterIP != "None" {
+				t.Errorf("expected clusterIP \"None\", got %q", createdSvc.Spec.ClusterIP)
+			}
+			if !reflect.DeepEqual(createdSvc.Spec.ClusterIPs, []string{"None"}) {
+				t.Errorf("expected clusterIPs [\"None\"], got %q", createdSvc.Spec.ClusterIPs)
+			}
+		})
+	}
+}

From 46d72896550a109f82577ef5fb8888de958129da Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 2 Jul 2021 23:10:44 -0700
Subject: [PATCH 19/79] Svc REST: Remove redundant Create tests

These cases are all covered in storage_test.
---
 .../core/service/storage/rest_test.go         | 1804 +----------------
 1 file changed, 6 insertions(+), 1798 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index edcc8157bcb..94609e50b5b 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -288,219 +288,6 @@ func makeIPNet6(t *testing.T) *net.IPNet {
 	return net
 }
 
-func TestServiceRegistryCreate(t *testing.T) {
-	testCases := []struct {
-		svc             *api.Service
-		name            string
-		families        []api.IPFamily
-		enableDualStack bool
-	}{{
-		name:            "Service IPFamily default cluster dualstack:off",
-		enableDualStack: false,
-		families:        []api.IPFamily{api.IPv4Protocol},
-		svc:             svctest.MakeService("foo"),
-	}, {
-		name:            "Service IPFamily:v4 dualstack off",
-		enableDualStack: false,
-		families:        []api.IPFamily{api.IPv4Protocol},
-		svc:             svctest.MakeService("foo", svctest.SetIPFamilies(api.IPv4Protocol)),
-	}, {
-		name:            "Service IPFamily:v4 dualstack on",
-		enableDualStack: true,
-		families:        []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-		svc:             svctest.MakeService("foo", svctest.SetIPFamilies(api.IPv4Protocol)),
-	}, {
-		name:            "Service IPFamily:v6 dualstack on",
-		enableDualStack: true,
-		families:        []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-		svc:             svctest.MakeService("foo", svctest.SetIPFamilies(api.IPv6Protocol)),
-	}}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			storage, server := NewTestREST(t, tc.families)
-			defer server.Terminate(t)
-
-			ctx := genericapirequest.NewDefaultContext()
-			createdSvc, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("error creating service %v", err)
-			}
-			createdService := createdSvc.(*api.Service)
-			objMeta, err := meta.Accessor(createdService)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if !metav1.HasObjectMetaSystemFieldValues(objMeta) {
-				t.Errorf("storage did not populate object meta field values")
-			}
-			if createdService.Name != "foo" {
-				t.Errorf("Expected foo, but got %v", createdService.Name)
-			}
-			if createdService.CreationTimestamp.IsZero() {
-				t.Errorf("Expected timestamp to be set, got: %v", createdService.CreationTimestamp)
-			}
-
-			for i, family := range createdService.Spec.IPFamilies {
-				allocator := storage.alloc.serviceIPAllocatorsByFamily[family]
-				c := allocator.CIDR()
-				cidr := &c
-				if !cidr.Contains(netutils.ParseIPSloppy(createdService.Spec.ClusterIPs[i])) {
-					t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIPs[i])
-				}
-			}
-			srv, err := getService(storage, ctx, tc.svc.Name, &metav1.GetOptions{})
-			if err != nil {
-				t.Errorf("unexpected error: %v", err)
-			}
-			if srv == nil {
-				t.Errorf("Failed to find service: %s", tc.svc.Name)
-			}
-		})
-	}
-}
-
-func TestServiceRegistryCreateDryRun(t *testing.T) {
-	testCases := []struct {
-		name            string
-		svc             *api.Service
-		enableDualStack bool
-	}{{
-		name:            "v4 service featuregate off",
-		enableDualStack: false,
-		svc:             svctest.MakeService("foo", svctest.SetClusterIPs("1.2.3.4")),
-	}, {
-		name:            "v6 service featuregate on but singlestack",
-		enableDualStack: true,
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilies(api.IPv6Protocol),
-			svctest.SetClusterIPs("2000::1")),
-	}, {
-		name:            "dualstack v4,v6 service",
-		enableDualStack: true,
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-			svctest.SetClusterIPs("1.2.3.4", "2000::1")),
-	}, {
-		name:            "dualstack v6,v4 service",
-		enableDualStack: true,
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-			svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
-			svctest.SetClusterIPs("2000::1", "1.2.3.4")),
-	}}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
-
-			families := []api.IPFamily{api.IPv4Protocol}
-			if tc.enableDualStack {
-				families = append(families, api.IPv6Protocol)
-			}
-			storage, server := NewTestREST(t, families)
-			defer server.Terminate(t)
-
-			ctx := genericapirequest.NewDefaultContext()
-			_, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
-			if err != nil {
-				t.Fatalf("Unexpected error: %v", err)
-			}
-
-			for i, family := range tc.svc.Spec.IPFamilies {
-				alloc := storage.alloc.serviceIPAllocatorsByFamily[family]
-				if ipIsAllocated(t, alloc, tc.svc.Spec.ClusterIPs[i]) {
-					t.Errorf("unexpected side effect: ip allocated %v", tc.svc.Spec.ClusterIPs[i])
-				}
-			}
-
-			_, err = getService(storage, ctx, tc.svc.Name, &metav1.GetOptions{})
-			if err == nil {
-				t.Errorf("expected error")
-			}
-		})
-	}
-}
-
-func TestDryRunNodePort(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	// Test dry run create request with a node port
-	svc := svctest.MakeService("foo", svctest.SetTypeNodePort)
-	ctx := genericapirequest.NewDefaultContext()
-
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	createdSvc := obj.(*api.Service)
-	if createdSvc.Spec.Ports[0].NodePort == 0 {
-		t.Errorf("expected NodePort value assigned")
-	}
-	if portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
-		t.Errorf("unexpected side effect: NodePort allocated")
-	}
-	_, err = getService(storage, ctx, svc.Name, &metav1.GetOptions{})
-	if err == nil {
-		// Should get a not-found.
-		t.Errorf("expected error")
-	}
-
-	// Test dry run create request with multi node port
-	svc = svctest.MakeService("foo",
-		svctest.SetTypeNodePort,
-		svctest.SetPorts(
-			svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6503), api.ProtocolTCP),
-			svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6503), api.ProtocolUDP)),
-		svctest.SetNodePorts(30053, 30053))
-	expectNodePorts := collectServiceNodePorts(svc)
-	obj, err = storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	createdSvc = obj.(*api.Service)
-	actualNodePorts := collectServiceNodePorts(createdSvc)
-	if !reflect.DeepEqual(actualNodePorts, expectNodePorts) {
-		t.Errorf("Expected %v, but got %v", expectNodePorts, actualNodePorts)
-	}
-	for i := range svc.Spec.Ports {
-		if portIsAllocated(t, storage.alloc.serviceNodePorts, svc.Spec.Ports[i].NodePort) {
-			t.Errorf("unexpected side effect: NodePort allocated")
-		}
-	}
-	_, err = getService(storage, ctx, svc.Name, &metav1.GetOptions{})
-	if err == nil {
-		// Should get a not-found.
-		t.Errorf("expected error")
-	}
-
-	// Test dry run create request with multiple unspecified node ports,
-	// so PortAllocationOperation.AllocateNext() will be called multiple times.
-	svc = svctest.MakeService("foo",
-		svctest.SetTypeNodePort,
-		svctest.SetPorts(
-			svctest.MakeServicePort("port-a", 53, intstr.FromInt(6503), api.ProtocolTCP),
-			svctest.MakeServicePort("port-b", 54, intstr.FromInt(6504), api.ProtocolUDP)))
-	obj, err = storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	createdSvc = obj.(*api.Service)
-	actualNodePorts = collectServiceNodePorts(createdSvc)
-	if len(actualNodePorts) != len(svc.Spec.Ports) {
-		t.Fatalf("Expected service to have %d ports, but got %v", len(svc.Spec.Ports), actualNodePorts)
-	}
-	seen := map[int]bool{}
-	for _, np := range actualNodePorts {
-		if seen[np] {
-			t.Errorf("Expected unique port numbers, but got %v", actualNodePorts)
-		} else {
-			seen[np] = true
-		}
-	}
-}
-
 func TestServiceRegistryCreateMultiNodePortsService(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
@@ -574,28 +361,6 @@ func TestServiceRegistryCreateMultiNodePortsService(t *testing.T) {
 	}
 }
 
-func TestServiceStorageValidatesCreate(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	failureCases := map[string]*api.Service{
-		"empty ID": svctest.MakeService(""),
-		"empty port": svctest.MakeService("foo", svctest.SetPorts(
-			svctest.MakeServicePort("p", 0, intstr.FromInt(80), api.ProtocolTCP))),
-		"missing targetPort": svctest.MakeService("foo", svctest.SetPorts(
-			svctest.MakeServicePort("p", 80, intstr.IntOrString{}, api.ProtocolTCP))),
-	}
-	ctx := genericapirequest.NewDefaultContext()
-	for _, failureCase := range failureCases {
-		c, err := storage.Create(ctx, failureCase, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-		if c != nil {
-			t.Errorf("Expected nil object")
-		}
-		if !errors.IsInvalid(err) {
-			t.Errorf("Expected to get an invalid resource error, got %v", err)
-		}
-	}
-}
-
 func TestServiceRegistryUpdate(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
@@ -996,123 +761,6 @@ func TestServiceStorageValidatesUpdate(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryLoadBalancerService(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("foo", svctest.SetTypeLoadBalancer)
-	_, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Errorf("Failed to create service: %#v", err)
-	}
-	srv, err := getService(storage, ctx, svc.Name, &metav1.GetOptions{})
-	if err != nil {
-		t.Errorf("Unexpected error: %v", err)
-	}
-	if srv == nil {
-		t.Fatalf("Failed to find service: %s", svc.Name)
-	}
-	serviceNodePorts := collectServiceNodePorts(srv)
-	if len(serviceNodePorts) == 0 {
-		t.Errorf("Failed to find NodePorts of service : %s", srv.Name)
-	}
-}
-
-func TestAllocateLoadBalancerNodePorts(t *testing.T) {
-	testcases := []struct {
-		name                 string
-		svc                  *api.Service
-		expectNodePorts      bool
-		allocateNodePortGate bool
-		expectError          bool
-	}{{
-		name: "allocate false, gate on, not specified",
-		svc: svctest.MakeService("alloc-false",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetAllocateLoadBalancerNodePorts(false)),
-		expectNodePorts:      false,
-		allocateNodePortGate: true,
-	}, {
-		name: "allocate true, gate on, not specified",
-		svc: svctest.MakeService("alloc-true",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetAllocateLoadBalancerNodePorts(true)),
-		expectNodePorts:      true,
-		allocateNodePortGate: true,
-	}, {
-		name: "allocate false, gate on, port specified",
-		svc: svctest.MakeService("alloc-false-specific",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetUniqueNodePorts,
-			svctest.SetAllocateLoadBalancerNodePorts(false)),
-		expectNodePorts:      true,
-		allocateNodePortGate: true,
-	}, {
-		name: "allocate true, gate on, port specified",
-		svc: svctest.MakeService("alloc-true-specific",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetUniqueNodePorts,
-			svctest.SetAllocateLoadBalancerNodePorts(true)),
-		expectNodePorts:      true,
-		allocateNodePortGate: true,
-	}, {
-		name: "allocate nil, gate off",
-		svc: svctest.MakeService("alloc-nil",
-			svctest.SetTypeLoadBalancer,
-			func(s *api.Service) {
-				s.Spec.AllocateLoadBalancerNodePorts = nil
-			}),
-		expectNodePorts:      true,
-		allocateNodePortGate: false,
-	}, {
-		name: "allocate false, gate off",
-		svc: svctest.MakeService("alloc-false",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetAllocateLoadBalancerNodePorts(false)),
-		expectNodePorts:      true,
-		allocateNodePortGate: false,
-	}, {
-		name: "allocate true, gate off",
-		svc: svctest.MakeService("alloc-true",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetAllocateLoadBalancerNodePorts(true)),
-		expectNodePorts:      true,
-		allocateNodePortGate: false,
-	}}
-	for _, tc := range testcases {
-		t.Run(tc.name, func(t *testing.T) {
-			ctx := genericapirequest.NewDefaultContext()
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLBNodePortControl, tc.allocateNodePortGate)()
-
-			storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-			defer server.Terminate(t)
-
-			_, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if err != nil {
-				if tc.expectError {
-					return
-				}
-				t.Errorf("failed to create service: %#v", err)
-			}
-			srv, err := getService(storage, ctx, tc.svc.Name, &metav1.GetOptions{})
-			if err != nil {
-				t.Errorf("unexpected error: %v", err)
-			}
-			if srv == nil {
-				t.Fatalf("failed to find service: %s", tc.svc.Name)
-			}
-			serviceNodePorts := collectServiceNodePorts(srv)
-			if (len(serviceNodePorts) != 0) != tc.expectNodePorts {
-				exp := "0"
-				if tc.expectNodePorts {
-					exp = ">0"
-				}
-				t.Errorf("allocated NodePorts not as expected: expected %v, got %v", exp, len(serviceNodePorts))
-			}
-		})
-	}
-}
-
 func TestServiceRegistryDelete(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
@@ -1470,7 +1118,7 @@ func TestServiceRegistryList(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryIPAllocation(t *testing.T) {
+func TestServiceRegistryCreateIPAllocation(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 
@@ -1523,7 +1171,7 @@ func TestServiceRegistryIPAllocation(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryIPReallocation(t *testing.T) {
+func TestServiceRegistryCreateIPReallocation(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 
@@ -1611,36 +1259,9 @@ func TestServiceRegistryIPUpdate(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryIPLoadBalancer(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	svc := svctest.MakeService("foo", svctest.SetTypeLoadBalancer)
-	ctx := genericapirequest.NewDefaultContext()
-	createdSvc, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if createdSvc == nil || err != nil {
-		t.Errorf("Unexpected failure creating service %v", err)
-	}
-
-	createdService := createdSvc.(*api.Service)
-	if createdService.Spec.Ports[0].Port != svc.Spec.Ports[0].Port {
-		t.Errorf("Expected port %d, but got %v", svc.Spec.Ports[0].Port, createdService.Spec.Ports[0].Port)
-	}
-	if !makeIPNet(t).Contains(netutils.ParseIPSloppy(createdService.Spec.ClusterIPs[0])) {
-		t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIPs[0])
-	}
-
-	update := createdService.DeepCopy()
-
-	_, _, err = storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-	if err != nil {
-		t.Errorf("Unexpected error %v", err)
-	}
-}
-
 // Validate allocation of a nodePort when ExternalTrafficPolicy is set to Local
 // and type is LoadBalancer.
-func TestServiceRegistryExternalTrafficHealthCheckNodePortAllocation(t *testing.T) {
+func TestServiceRegistryCreateExternalTrafficHealthCheckNodePortAllocation(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
@@ -1700,7 +1321,7 @@ func TestServiceRegistryExternalTrafficHealthCheckNodePortUserAllocation(t *test
 }
 
 // Validate that the service creation fails when the requested port number is -1.
-func TestServiceRegistryExternalTrafficHealthCheckNodePortNegative(t *testing.T) {
+func TestServiceRegistryCreateExternalTrafficHealthCheckNodePortNegative(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
@@ -1716,7 +1337,7 @@ func TestServiceRegistryExternalTrafficHealthCheckNodePortNegative(t *testing.T)
 }
 
 // Validate that the health check nodePort is not allocated when ExternalTrafficPolicy is set to Global.
-func TestServiceRegistryExternalTrafficGlobal(t *testing.T) {
+func TestServiceRegistryCreateExternalTrafficGlobal(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
@@ -1806,248 +1427,7 @@ func TestServiceRegistryInternalTrafficPolicyLocalThenCluster(t *testing.T) {
 	}
 }
 
-func TestInitClusterIP(t *testing.T) {
-	testCases := []struct {
-		name string
-		svc  *api.Service
-
-		enableDualStackAllocator bool
-		preAllocateClusterIPs    map[api.IPFamily]string
-		expectError              bool
-		expectedCountIPs         int
-		expectedClusterIPs       []string
-	}{{
-		name:                     "Allocate single stack ClusterIP (v4)",
-		svc:                      svctest.MakeService("foo"),
-		enableDualStackAllocator: false,
-		expectError:              false,
-		preAllocateClusterIPs:    nil,
-		expectedCountIPs:         1,
-	}, {
-		name: "Allocate single ClusterIP (v6)",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilies(api.IPv6Protocol)),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		preAllocateClusterIPs:    nil,
-		expectedCountIPs:         1,
-	}, {
-		name: "Allocate specified ClusterIP (v4)",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilies(api.IPv4Protocol),
-			svctest.SetClusterIPs("1.2.3.4")),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		preAllocateClusterIPs:    nil,
-		expectedCountIPs:         1,
-		expectedClusterIPs:       []string{"1.2.3.4"},
-	}, {
-		name: "Allocate specified ClusterIP-v6",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilies(api.IPv6Protocol),
-			svctest.SetClusterIPs("2000:0:0:0:0:0:0:1")),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         1,
-		expectedClusterIPs:       []string{"2000:0:0:0:0:0:0:1"},
-	}, {
-		name: "Allocate dual stack - on a non dual stack ",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilies(api.IPv4Protocol)),
-		expectError:              false,
-		enableDualStackAllocator: false,
-		expectedCountIPs:         1,
-	}, {
-		name: "Allocate dual stack - upgrade - v4, v6",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-			svctest.SetIPFamilies(api.IPv4Protocol)),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         2,
-	}, {
-		name: "Allocate dual stack - upgrade - v4, v6 - specific first IP",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-			svctest.SetIPFamilies(api.IPv4Protocol),
-			svctest.SetClusterIPs("1.2.3.4")),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         2,
-		expectedClusterIPs:       []string{"1.2.3.4"},
-	}, {
-		name: "Allocate dual stack - upgrade - v6, v4",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-			svctest.SetIPFamilies(api.IPv6Protocol)),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         2,
-	}, {
-		name: "Allocate dual stack - v4, v6 - specific ips",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-			svctest.SetClusterIPs("1.2.3.4", "2000:0:0:0:0:0:0:1")),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         2,
-		expectedClusterIPs:       []string{"1.2.3.4", "2000:0:0:0:0:0:0:1"},
-	}, {
-		name: "Allocate dual stack - upgrade - v6, v4 - specific ips",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-			svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
-			svctest.SetClusterIPs("2000:0:0:0:0:0:0:1", "1.2.3.4")),
-		expectError:              false,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         2,
-		expectedClusterIPs:       []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"},
-	}, {
-		name: "Shouldn't allocate ClusterIP",
-		svc: svctest.MakeService("foo",
-			svctest.SetClusterIPs("None")),
-		expectError:              false,
-		enableDualStackAllocator: false,
-		expectedCountIPs:         0,
-	}, {
-		name: "single stack, ip is pre allocated (ipv4)",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
-			svctest.SetIPFamilies(api.IPv4Protocol),
-			svctest.SetClusterIPs("1.2.3.4")),
-		expectError:              true,
-		enableDualStackAllocator: false,
-		expectedCountIPs:         0,
-		preAllocateClusterIPs:    map[api.IPFamily]string{api.IPv4Protocol: "1.2.3.4"},
-	}, {
-		name: "single stack, ip is pre allocated (ipv6)",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
-			svctest.SetIPFamilies(api.IPv6Protocol),
-			svctest.SetClusterIPs("2000:0:0:0:0:0:0:1")),
-		expectError:              true,
-		enableDualStackAllocator: true, // ipv6 allocator is always the second one during test
-		expectedCountIPs:         0,
-		preAllocateClusterIPs:    map[api.IPFamily]string{api.IPv6Protocol: "2000:0:0:0:0:0:0:1"},
-	}, {
-		name: "Allocate dual stack - upgrade - v6, v4 - specific ips (first ip can't be allocated)",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-			svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
-			svctest.SetClusterIPs("2000:0:0:0:0:0:0:1", "1.2.3.4")),
-		expectError:              true,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         0,
-		preAllocateClusterIPs:    map[api.IPFamily]string{api.IPv6Protocol: "2000:0:0:0:0:0:0:1"},
-	}, {
-		name: "Allocate dual stack - upgrade - v6, v4 - specific ips (second ip can't be allocated)",
-		svc: svctest.MakeService("foo",
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-			svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
-			svctest.SetClusterIPs("2000:0:0:0:0:0:0:1", "1.2.3.4")),
-		expectError:              true,
-		enableDualStackAllocator: true,
-		expectedCountIPs:         0,
-		preAllocateClusterIPs:    map[api.IPFamily]string{api.IPv4Protocol: "1.2.3.4"},
-	}}
-
-	for _, test := range testCases {
-		t.Run(test.name, func(t *testing.T) {
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-
-			// create the rest stack
-			families := []api.IPFamily{api.IPv4Protocol}
-			if test.enableDualStackAllocator {
-				families = append(families, api.IPv6Protocol)
-			}
-			storage, server := NewTestREST(t, families)
-			defer server.Terminate(t)
-
-			copySvc := test.svc.DeepCopy()
-
-			// pre allocate ips if any
-			for family, ip := range test.preAllocateClusterIPs {
-				allocator, ok := storage.alloc.serviceIPAllocatorsByFamily[family]
-				if !ok {
-					t.Fatalf("test is incorrect, allocator does not exist on rest")
-				}
-				if err := allocator.Allocate(netutils.ParseIPSloppy(ip)); err != nil {
-					t.Fatalf("test is incorrect, allocator failed to pre allocate IP with error:%v", err)
-				}
-			}
-			ctx := genericapirequest.NewDefaultContext()
-			createdSvc, err := storage.Create(ctx, test.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if test.expectError && err == nil {
-				t.Fatalf("error was expected, but no error was returned")
-			}
-
-			if !test.expectError && err != nil {
-				t.Fatalf("error was not expected, but got error %v", err)
-			}
-
-			if err != nil {
-				return // no more testing needed for this case
-			}
-			newSvc := createdSvc.(*api.Service)
-			isValidClusterIPFields(t, storage, copySvc, newSvc)
-
-			// if it has ips then let us check they have been correctly allocated
-			if newSvc.Spec.ClusterIPs[0] != api.ClusterIPNone {
-				for _, ip := range newSvc.Spec.ClusterIPs {
-					family := api.IPv4Protocol
-					if netutils.IsIPv6String(ip) {
-						family = api.IPv6Protocol
-					}
-					allocator := storage.alloc.serviceIPAllocatorsByFamily[family]
-					if !ipIsAllocated(t, allocator, ip) {
-						t.Fatalf("expected ip:%v to be allocated by %v allocator. it was not", ip, family)
-					}
-				}
-			}
-
-			allocatedIPs := 0
-			for _, ip := range newSvc.Spec.ClusterIPs {
-				if ip != api.ClusterIPNone {
-					allocatedIPs++
-				}
-			}
-
-			if allocatedIPs != test.expectedCountIPs {
-				t.Fatalf("incorrect allocated IP count expected %v got %v", test.expectedCountIPs, allocatedIPs)
-			}
-
-			for i, ip := range test.expectedClusterIPs {
-				if i >= len(newSvc.Spec.ClusterIPs) {
-					t.Fatalf("incorrect ips were assigne. expected to find %+v in %+v",
-						ip, newSvc.Spec.ClusterIPs)
-				}
-
-				if ip != newSvc.Spec.ClusterIPs[i] {
-					t.Fatalf("incorrect ips were assigne. expected to find %+v == %+v at position %v",
-						ip, newSvc.Spec.ClusterIPs[i], i)
-				}
-			}
-
-			// the following apply only on dual stack
-			if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-				return
-			}
-
-			shouldUpgrade := len(newSvc.Spec.IPFamilies) == 2 && *(newSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.alloc.serviceIPAllocatorsByFamily) == 2
-			if shouldUpgrade && len(newSvc.Spec.ClusterIPs) < 2 {
-				t.Fatalf("Service should have been upgraded %+v", newSvc)
-			}
-
-			if !shouldUpgrade && len(newSvc.Spec.ClusterIPs) > 1 {
-				t.Fatalf("Service should not have been upgraded %+v", newSvc)
-			}
-
-		})
-	}
-}
-
-func TestInitNodePorts(t *testing.T) {
+func TestCreateInitNodePorts(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
 	nodePortOp := portallocator.StartOperation(storage.alloc.serviceNodePorts, false)
@@ -2585,1178 +1965,6 @@ func TestServiceDowngrade(t *testing.T) {
 	}
 }
 
-func TestDefaultingValidation(t *testing.T) {
-	singleStack := api.IPFamilyPolicySingleStack
-	preferDualStack := api.IPFamilyPolicyPreferDualStack
-	requireDualStack := api.IPFamilyPolicyRequireDualStack
-
-	// takes in REST and modify it for a specific config
-	fnMakeSingleStackIPv4Allocator := func(rest *REST) {
-		rest.alloc.defaultServiceIPFamily = api.IPv4Protocol
-		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv4Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol]}
-	}
-
-	fnMakeSingleStackIPv6Allocator := func(rest *REST) {
-		rest.alloc.defaultServiceIPFamily = api.IPv6Protocol
-		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv6Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol]}
-	}
-
-	fnMakeDualStackStackIPv4IPv6Allocator := func(rest *REST) {
-		rest.alloc.defaultServiceIPFamily = api.IPv4Protocol
-		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{
-			api.IPv6Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol],
-			api.IPv4Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol],
-		}
-	}
-
-	fnMakeDualStackStackIPv6IPv4Allocator := func(rest *REST) {
-		rest.alloc.defaultServiceIPFamily = api.IPv6Protocol
-		rest.alloc.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{
-			api.IPv6Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol],
-			api.IPv4Protocol: rest.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol],
-		}
-	}
-
-	testCases := []struct {
-		name       string
-		modifyRest func(rest *REST)
-		oldSvc     *api.Service
-		svc        *api.Service
-
-		expectedIPFamilyPolicy *api.IPFamilyPolicyType
-		expectedIPFamilies     []api.IPFamily
-		expectError            bool
-	}{
-		////////////////////////////
-		// cluster configured as single stack v4
-		////////////////////////////
-		{
-			name:                   "[singlestack:v4] set: externalname on a single stack - v4",
-			modifyRest:             fnMakeSingleStackIPv4Allocator,
-			svc:                    svctest.MakeService("foo", svctest.SetTypeExternalName),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            false,
-		},
-		{
-			name:                   "[singlestack:v4] set: nothing",
-			modifyRest:             fnMakeSingleStackIPv4Allocator,
-			svc:                    svctest.MakeService("foo"),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-
-		{
-			name:       "[singlestack:v4] set: v4Cluster IPSet",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: v4IPFamilySet",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: v4IPFamilySet",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("10.0.0.4"),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: PreferDualStack",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: PreferDualStack + v4ClusterIPSet",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: PreferDualStack + v4ClusterIPSet + v4FamilySet",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: v6IPSet",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v4] set: v6IPFamily",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v4] set: RequireDualStack",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v4] set: RequireDualStack + family",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		// selector less
-		{
-			name:       "[singlestack:v4] set: selectorless, families are ignored",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: selectorless, no families",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: selectorless, user selected",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v4] set: selectorless, user set to preferDualStack",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		// tests incorrect setting for IPFamilyPolicy
-		{
-			name:       "[singlestack:v4] set: multifamily set to preferDualStack",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v4] set: multifamily set to singleStack",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v4] set: mult clusterips set to preferDualStack",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v4] set: multi clusterips set to singleStack",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-
-		////////////////////////////
-		// cluster configured as single stack v6
-		////////////////////////////
-		{
-			name:                   "[singlestack:v6] set: externalname on a single stack - v4",
-			modifyRest:             fnMakeSingleStackIPv6Allocator,
-			svc:                    svctest.MakeService("foo", svctest.SetTypeExternalName),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            false,
-		},
-		{
-			name:                   "[singlestack:v6] set: nothing",
-			modifyRest:             fnMakeSingleStackIPv6Allocator,
-			svc:                    svctest.MakeService("foo"),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: v6Cluster IPSet",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: v4IPFamilySet",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: v6IPFamilySet",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("2000::1"),
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: PreferDualStack",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: PreferDualStack + v6ClusterIPSet",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: PreferDualStack + v6ClusterIPSet + v6FamilySet",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetIPFamilies(api.IPv6Protocol),
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: v4IPSet",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("10.0.0.10")),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v6] set: v4IPFamily",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v6] set: RequireDualStack (on single stack ipv6 cluster)",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v6] set: RequireDualStack + family (on single stack ipv6 cluster)",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-
-		// selector less
-		{
-			name:       "[singlestack:v6] set: selectorless, families are ignored",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: selectorless, no families",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: selectorless, user selected",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[singlestack:v6] set: selectorless, user set to preferDualStack",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		// tests incorrect setting for IPFamilyPolicy
-		{
-			name:       "[singlestack:v6] set: multifamily set to preferDualStack",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v6] set: multifamily set to singleStack",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v6] set: mult clusterips set to preferDualStack",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[singlestack:v6] set: multi clusterips set to singleStack",
-			modifyRest: fnMakeSingleStackIPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-
-		////////////////////////////
-		// cluster configured as dual stack v4,6
-		////////////////////////////
-		{
-			name:                   "[dualstack:v4,v6] set: externalname on a dual stack - v4,v6",
-			modifyRest:             fnMakeDualStackStackIPv4IPv6Allocator,
-			svc:                    svctest.MakeService("foo", svctest.SetTypeExternalName),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            false,
-		},
-		{
-			name:                   "[dualstack:v4,v6] set: nothing",
-			modifyRest:             fnMakeDualStackStackIPv4IPv6Allocator,
-			svc:                    svctest.MakeService("foo"),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-
-		{
-			name:       "[dualstack:v4,v6] set: v4ClusterIPSet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: v4IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: v4IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("10.0.0.4"),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: v6ClusterIPSet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: v6IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: v6IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("2000::1"),
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		// prefer dual stack
-		{
-			name:       "[dualstack:v4,v6] set: PreferDualStack.",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: PreferDualStack + v4ClusterIPSet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: PreferDualStack + v4ClusterIPSet + v4FamilySet",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		// require dual stack
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + family v4",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + family v6",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + family +ip v4",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10"),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			//
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + family +ip v6",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1"),
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + ip v6",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + ip v4",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + ips",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10", "2000::1")),
-			//
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + ips",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1", "10.0.0.10")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: RequireDualStack + ips + families v6,v4",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1", "10.0.0.10"),
-				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ips + families v4,v6",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10", "2000::1"),
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,v6] set: selectorless, no families",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,6] set: selectorless, user selected",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,6] set: selectorless, user set to prefer",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		// tests incorrect setting for IPFamilyPolicy
-		{
-			name:       "[duakstack:v4,6] set: multifamily set to preferDualStack",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,6] set: multifamily set to singleStack",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[dualstack:v4,6] set: mult clusterips set to preferDualStack",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,6] set: multi clusterips set to singleStack",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-
-		////////////////////////////
-		// cluster configured as dual stack v6,4
-		////////////////////////////
-		{
-			name:                   "[dualstack:v6,v4] set: externalname on a dual stack - v6,v4",
-			modifyRest:             fnMakeDualStackStackIPv6IPv4Allocator,
-			svc:                    svctest.MakeService("foo", svctest.SetTypeExternalName),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            false,
-		},
-		{
-			name:                   "[dualstack:v6,v4] set: nothing",
-			modifyRest:             fnMakeDualStackStackIPv6IPv4Allocator,
-			svc:                    svctest.MakeService("foo"),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: v4ClusterIPSet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: v4IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			//
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: v4IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("10.0.0.4"),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: v6ClusterIPSet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: v6IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: v6IPFamilySet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("2000::1"),
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		// prefer dual stack
-		{
-			name:       "[dualstack:v6,v4] set: PreferDualStack.",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: PreferDualStack + v4ClusterIPSet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: PreferDualStack + v4ClusterIPSet + v4FamilySet",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetClusterIPs("10.0.0.4")),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		// require dual stack
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + family v4",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + family v6",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + family +ip v4",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10"),
-				svctest.SetIPFamilies(api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + family +ip v6",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1"),
-				svctest.SetIPFamilies(api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ip v6",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ip v4",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ip v4",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ips",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10", "2000::1")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ips",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1", "10.0.0.10")),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ips + families v6,v4",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("2000::1", "10.0.0.10"),
-				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: RequireDualStack + ips + families v4,v6",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-				svctest.SetClusterIPs("10.0.0.10", "2000::1"),
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: selectorless, no families",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				func(s *api.Service) { s.Spec.Selector = nil }),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: selectorless, user selected",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,v4] set: selectorless, user set to prefer",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("None"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			expectError:            false,
-		},
-		// tests incorrect setting for IPFamilyPolicy
-		{
-			name:       "[duakstack:v6,5] set: multifamily set to preferDualStack",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v4,6] set: multifamily set to singleStack",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-		{
-			name:       "[dualstack:v6,4] set: mult clusterips set to preferDualStack",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-		{
-			name:       "[dualstack:v6,4] set: multi clusterips set to singleStack",
-			modifyRest: fnMakeDualStackStackIPv6IPv4Allocator,
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: nil,
-			expectedIPFamilies:     nil,
-			expectError:            true,
-		},
-
-		// preferDualStack services should not be updated
-		// to match cluster config if the user didn't change any
-		// ClusterIPs related fields
-		{
-			name:       "unchanged preferDualStack-1-ClusterUpgraded",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			oldSvc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1"),
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1"),
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-
-		{
-			name:       "unchanged preferDualStack-2-ClusterDowngraded",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			oldSvc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-			expectedIPFamilyPolicy: &preferDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-
-		{
-			name:       "changed preferDualStack-1 (cluster upgraded)",
-			modifyRest: fnMakeDualStackStackIPv4IPv6Allocator,
-			oldSvc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1"),
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-
-			svc: svctest.MakeService("foo",
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
-			expectedIPFamilyPolicy: &requireDualStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			expectError:            false,
-		},
-
-		{
-			name:       "changed preferDualStack-2-ClusterDowngraded",
-			modifyRest: fnMakeSingleStackIPv4Allocator,
-			oldSvc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1", "2001::1"),
-				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
-
-			svc: svctest.MakeService("foo",
-				svctest.SetClusterIPs("1.1.1.1"),
-				svctest.SetIPFamilies(api.IPv4Protocol),
-				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
-			expectedIPFamilyPolicy: &singleStack,
-			expectedIPFamilies:     []api.IPFamily{api.IPv4Protocol},
-			expectError:            false,
-		},
-	}
-
-	// This func only runs when feature gate is on
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
-	defer server.Terminate(t)
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-
-			// reset to defaults
-			fnMakeDualStackStackIPv4IPv6Allocator(storage)
-			// optionally apply test-specific changes
-			if testCase.modifyRest != nil {
-				testCase.modifyRest(storage)
-			}
-
-			err := storage.alloc.initIPFamilyFields(testCase.oldSvc, testCase.svc)
-			if err != nil && !testCase.expectError {
-				t.Fatalf("error %v was not expected", err)
-			}
-
-			if err == nil && testCase.expectError {
-				t.Fatalf("error was expected, but no error returned")
-			}
-
-			if err != nil {
-				t.Logf("test concluded successfully with terminal error %v", err)
-				return
-			}
-
-			// IPFamily Policy
-			if (testCase.expectedIPFamilyPolicy == nil && testCase.svc.Spec.IPFamilyPolicy != nil) ||
-				(testCase.expectedIPFamilyPolicy != nil && testCase.svc.Spec.IPFamilyPolicy == nil) {
-				t.Fatalf("ipFamilyPolicy expected:%v got %v", testCase.expectedIPFamilyPolicy, testCase.svc.Spec.IPFamilyPolicy)
-			}
-
-			if testCase.expectedIPFamilyPolicy != nil {
-				if *testCase.expectedIPFamilyPolicy != *testCase.svc.Spec.IPFamilyPolicy {
-					t.Fatalf("ipFamilyPolicy expected:%s got %s", *testCase.expectedIPFamilyPolicy, *testCase.svc.Spec.IPFamilyPolicy)
-				}
-			}
-
-			if len(testCase.expectedIPFamilies) != len(testCase.svc.Spec.IPFamilies) {
-				t.Fatalf("expected len of IPFamilies %v got %v", len(testCase.expectedIPFamilies), len(testCase.svc.Spec.IPFamilies))
-			}
-
-			// match families
-			for i, family := range testCase.expectedIPFamilies {
-				if testCase.svc.Spec.IPFamilies[i] != family {
-					t.Fatalf("expected ip family %v at %v got %v", family, i, testCase.svc.Spec.IPFamilies)
-				}
-			}
-		})
-	}
-}
-
 // validates that the service created, updated by REST
 // has correct ClusterIPs related fields
 func isValidClusterIPFields(t *testing.T, storage *REST, pre *api.Service, post *api.Service) {

From 2212924a96f30a787fdabff6493bd08a89eff4e4 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 3 Jul 2021 16:58:50 -0700
Subject: [PATCH 20/79] Svc REST: Better NodePort tests

This commit ports the NodePort test from rest_test to storage_test.
It's not a direct port, though.  I have added many more cases (much more
exhaustive) and more assertions.

This includes cases for gate MixedProtocolLBService.
---
 .../core/service/storage/rest_test.go         | 158 -------
 .../core/service/storage/storage_test.go      | 393 ++++++++++++++++++
 2 files changed, 393 insertions(+), 158 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 94609e50b5b..37fc6327cae 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -25,7 +25,6 @@ import (
 	"testing"
 
 	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/meta"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -288,79 +287,6 @@ func makeIPNet6(t *testing.T) *net.IPNet {
 	return net
 }
 
-func TestServiceRegistryCreateMultiNodePortsService(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	testCases := []struct {
-		svc             *api.Service
-		name            string
-		expectNodePorts []int
-	}{{
-		svc: svctest.MakeService("foo1",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6503), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6503), api.ProtocolUDP)),
-			svctest.SetNodePorts(30053, 30053)),
-		name:            "foo1",
-		expectNodePorts: []int{30053, 30053},
-	}, {
-		svc: svctest.MakeService("foo2",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 54, intstr.FromInt(6504), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 54, intstr.FromInt(6504), api.ProtocolUDP)),
-			svctest.SetNodePorts(30054, 30054)),
-		name:            "foo2",
-		expectNodePorts: []int{30054, 30054},
-	}, {
-		svc: svctest.MakeService("foo3",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 55, intstr.FromInt(6505), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 55, intstr.FromInt(6506), api.ProtocolUDP)),
-			svctest.SetNodePorts(30055, 30056)),
-		name:            "foo3",
-		expectNodePorts: []int{30055, 30056},
-	}}
-
-	ctx := genericapirequest.NewDefaultContext()
-	for _, test := range testCases {
-		createdSvc, err := storage.Create(ctx, test.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-		if err != nil {
-			t.Fatalf("Unexpected error: %v", err)
-		}
-		createdService := createdSvc.(*api.Service)
-		objMeta, err := meta.Accessor(createdService)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !metav1.HasObjectMetaSystemFieldValues(objMeta) {
-			t.Errorf("storage did not populate object meta field values")
-		}
-		if createdService.Name != test.name {
-			t.Errorf("Expected %s, but got %s", test.name, createdService.Name)
-		}
-		serviceNodePorts := collectServiceNodePorts(createdService)
-		if !reflect.DeepEqual(serviceNodePorts, test.expectNodePorts) {
-			t.Errorf("Expected %v, but got %v", test.expectNodePorts, serviceNodePorts)
-		}
-		srv, err := getService(storage, ctx, test.name, &metav1.GetOptions{})
-		if err != nil {
-			t.Errorf("unexpected error: %v", err)
-		}
-		if srv == nil {
-			t.Fatalf("Failed to find service: %s", test.name)
-		}
-		for i := range serviceNodePorts {
-			nodePort := serviceNodePorts[i]
-			// Release the node port at the end of the test case.
-			storage.alloc.serviceNodePorts.Release(nodePort)
-		}
-	}
-}
-
 func TestServiceRegistryUpdate(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
@@ -1427,90 +1353,6 @@ func TestServiceRegistryInternalTrafficPolicyLocalThenCluster(t *testing.T) {
 	}
 }
 
-func TestCreateInitNodePorts(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	nodePortOp := portallocator.StartOperation(storage.alloc.serviceNodePorts, false)
-
-	testCases := []struct {
-		name                     string
-		service                  *api.Service
-		expectSpecifiedNodePorts []int
-	}{{
-		name:                     "Service doesn't have specified NodePort",
-		service:                  svctest.MakeService("foo", svctest.SetTypeNodePort),
-		expectSpecifiedNodePorts: []int{},
-	}, {
-		name: "Service has one specified NodePort",
-		service: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP)),
-			svctest.SetNodePorts(30053)),
-		expectSpecifiedNodePorts: []int{30053},
-	}, {
-		name: "Service has two same ports with different protocols and specifies same NodePorts",
-		service: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30054, 30054)),
-		expectSpecifiedNodePorts: []int{30054, 30054},
-	}, {
-		name: "Service has two same ports with different protocols and specifies different NodePorts",
-		service: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30055, 30056)),
-		expectSpecifiedNodePorts: []int{30055, 30056},
-	}, {
-		name: "Service has two different ports with different protocols and specifies different NodePorts",
-		service: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 54, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30057, 30058)),
-		expectSpecifiedNodePorts: []int{30057, 30058},
-	}, {
-		name: "Service has two same ports with different protocols but only specifies one NodePort",
-		service: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30059)),
-		expectSpecifiedNodePorts: []int{30059, 30059},
-	}}
-
-	for _, test := range testCases {
-		err := initNodePorts(test.service, nodePortOp)
-		if err != nil {
-			t.Errorf("%q: unexpected error: %v", test.name, err)
-			continue
-		}
-
-		serviceNodePorts := collectServiceNodePorts(test.service)
-		if len(test.expectSpecifiedNodePorts) == 0 {
-			for _, nodePort := range serviceNodePorts {
-				if !storage.alloc.serviceNodePorts.Has(nodePort) {
-					t.Errorf("%q: unexpected NodePort %d, out of range", test.name, nodePort)
-				}
-			}
-		} else if !reflect.DeepEqual(serviceNodePorts, test.expectSpecifiedNodePorts) {
-			t.Errorf("%q: expected NodePorts %v, but got %v", test.name, test.expectSpecifiedNodePorts, serviceNodePorts)
-		}
-		for i := range serviceNodePorts {
-			nodePort := serviceNodePorts[i]
-			// Release the node port at the end of the test case.
-			storage.alloc.serviceNodePorts.Release(nodePort)
-		}
-	}
-}
-
 func TestUpdateNodePorts(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index cf1ac619cb2..52cfc42e210 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -4733,6 +4733,399 @@ func TestCreateInitIPFields(t *testing.T) {
 	}
 }
 
+func TestCreateInitNodePorts(t *testing.T) {
+	testCases := []struct {
+		name                         string
+		svc                          *api.Service
+		expectError                  bool
+		expectNodePorts              bool
+		gateMixedProtocolLBService   bool
+		gateServiceLBNodePortControl bool
+	}{{
+		name:            "type:ExternalName",
+		svc:             svctest.MakeService("foo"),
+		expectNodePorts: false,
+	}, {
+		name: "type:ExternalName_with_NodePorts",
+		svc: svctest.MakeService("foo",
+			svctest.SetUniqueNodePorts),
+		expectError: true,
+	}, {
+		name:            "type:ClusterIP",
+		svc:             svctest.MakeService("foo"),
+		expectNodePorts: false,
+	}, {
+		name: "type:ClusterIP_with_NodePorts",
+		svc: svctest.MakeService("foo",
+			svctest.SetUniqueNodePorts),
+		expectError: true,
+	}, {
+		name: "type:NodePort_single_port_unspecified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort),
+		expectNodePorts: true,
+	}, {
+		name: "type:NodePort_single_port_specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort, svctest.SetUniqueNodePorts),
+		expectNodePorts: true,
+	}, {
+		name: "type:NodePort_multiport_unspecified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+		expectNodePorts: true,
+	}, {
+		name: "type:NodePort_multiport_specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetUniqueNodePorts),
+		expectNodePorts: true,
+	}, {
+		name: "type:NodePort_multiport_same",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetNodePorts(30080, 30080)),
+		expectError: true,
+	}, {
+		name: "type:NodePort_multiport_multiproto_unspecified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP))),
+		expectNodePorts: true,
+	}, {
+		name: "type:NodePort_multiport_multiproto_specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP)),
+			svctest.SetUniqueNodePorts),
+		expectNodePorts: true,
+	}, {
+		name: "type:NodePort_multiport_multiproto_same",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP)),
+			svctest.SetNodePorts(30053, 30053)),
+		expectNodePorts: true,
+	}, {
+		name: "type:NodePort_multiport_multiproto_conflict",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 93, intstr.FromInt(93), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 76, intstr.FromInt(76), api.ProtocolUDP)),
+			svctest.SetNodePorts(30093, 30093)),
+		expectError: true,
+	}, {
+		// When the ServiceLBNodePortControl gate is locked, this can be removed.
+		name: "type:LoadBalancer_single_port_unspecified_gateServiceLBNodePortControl:off_alloc:nil",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer),
+		gateServiceLBNodePortControl: false,
+		expectNodePorts:              true,
+	}, {
+		// When the ServiceLBNodePortControl gate is locked, this can be removed.
+		name: "type:LoadBalancer_single_port_unspecified_gateServiceLBNodePortControl:off_alloc:false",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetAllocateLoadBalancerNodePorts(false)),
+		gateServiceLBNodePortControl: false,
+		expectNodePorts:              true,
+	}, {
+		// When the ServiceLBNodePortControl gate is locked, this can be removed.
+		name: "type:LoadBalancer_single_port_unspecified_gateServiceLBNodePortControl:off_alloc:true",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetAllocateLoadBalancerNodePorts(true)),
+		gateServiceLBNodePortControl: false,
+		expectNodePorts:              true,
+	}, {
+		// When the ServiceLBNodePortControl gate is locked, this can be removed.
+		name: "type:LoadBalancer_single_port_specified_gateServiceLBNodePortControl:off",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer, svctest.SetUniqueNodePorts),
+		gateServiceLBNodePortControl: false,
+		expectNodePorts:              true,
+	}, {
+		// When the ServiceLBNodePortControl gate is locked, this can be removed.
+		name: "type:LoadBalancer_multiport_unspecified_gateServiceLBNodePortControl:off",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+		gateServiceLBNodePortControl: false,
+		expectNodePorts:              true,
+	}, {
+		// When the ServiceLBNodePortControl gate is locked, this can be removed.
+		name: "type:LoadBalancer_multiport_specified_gateServiceLBNodePortControl:off",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetUniqueNodePorts),
+		gateServiceLBNodePortControl: false,
+		expectNodePorts:              true,
+	}, {
+		name: "type:LoadBalancer_single_port_unspecified_gateServiceLBNodePortControl:on_alloc:false",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetAllocateLoadBalancerNodePorts(false)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              false,
+	}, {
+		name: "type:LoadBalancer_single_port_unspecified_gateServiceLBNodePortControl:on_alloc:true",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetAllocateLoadBalancerNodePorts(true)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              true,
+	}, {
+		name: "type:LoadBalancer_single_port_specified_gateServiceLBNodePortControl:on_alloc:false",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetUniqueNodePorts,
+			svctest.SetAllocateLoadBalancerNodePorts(false)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              true,
+	}, {
+		name: "type:LoadBalancer_single_port_specified_gateServiceLBNodePortControl:on_alloc:true",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetUniqueNodePorts,
+			svctest.SetAllocateLoadBalancerNodePorts(true)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              true,
+	}, {
+		name: "type:LoadBalancer_multiport_unspecified_gateServiceLBNodePortControl:on_alloc:false",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetAllocateLoadBalancerNodePorts(false)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              false,
+	}, {
+		name: "type:LoadBalancer_multiport_unspecified_gateServiceLBNodePortControl:on_alloc:true",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetAllocateLoadBalancerNodePorts(true)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              true,
+	}, {
+		name: "type:LoadBalancer_multiport_specified_gateServiceLBNodePortControl:on_alloc:false",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetUniqueNodePorts,
+			svctest.SetAllocateLoadBalancerNodePorts(false)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              true,
+	}, {
+		name: "type:LoadBalancer_multiport_specified_gateServiceLBNodePortControl:on_alloc:true",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetUniqueNodePorts,
+			svctest.SetAllocateLoadBalancerNodePorts(true)),
+		gateServiceLBNodePortControl: true,
+		expectNodePorts:              true,
+	}, {
+		name: "type:LoadBalancer_multiport_same",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetNodePorts(30080, 30080)),
+		expectError: true,
+	}, {
+		// When the MixedProtocolLBService gate is locked, this can be removed.
+		name: "type:LoadBalancer_multiport_multiproto_unspecified_MixedProtocolLBService:off",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP))),
+		gateMixedProtocolLBService: false,
+		expectError:                true,
+	}, {
+		// When the MixedProtocolLBService gate is locked, this can be removed.
+		name: "type:LoadBalancer_multiport_multiproto_specified_MixedProtocolLBService:off",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP)),
+			svctest.SetUniqueNodePorts),
+		gateMixedProtocolLBService: false,
+		expectError:                true,
+	}, {
+		// When the MixedProtocolLBService gate is locked, this can be removed.
+		name: "type:LoadBalancer_multiport_multiproto_same_MixedProtocolLBService:off",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP)),
+			svctest.SetNodePorts(30053, 30053)),
+		gateMixedProtocolLBService: false,
+		expectError:                true,
+	}, {
+		name: "type:LoadBalancer_multiport_multiproto_unspecified_MixedProtocolLBService:on",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP))),
+		gateMixedProtocolLBService: true,
+		expectNodePorts:            true,
+	}, {
+		name: "type:LoadBalancer_multiport_multiproto_specified_MixedProtocolLBService:on",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP)),
+			svctest.SetUniqueNodePorts),
+		gateMixedProtocolLBService: true,
+		expectNodePorts:            true,
+	}, {
+		name: "type:LoadBalancer_multiport_multiproto_same_MixedProtocolLBService:on",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP)),
+			svctest.SetNodePorts(30053, 30053)),
+		gateMixedProtocolLBService: true,
+		expectNodePorts:            true,
+	}, {
+		name: "type:LoadBalancer_multiport_multiproto_conflict",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 93, intstr.FromInt(93), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 76, intstr.FromInt(76), api.ProtocolUDP)),
+			svctest.SetNodePorts(30093, 30093)),
+		expectError: true,
+	}}
+
+	// Do this in the outer scope for performance.
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	for _, tc := range testCases {
+		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLBNodePortControl, tc.gateServiceLBNodePortControl)()
+		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MixedProtocolLBService, tc.gateMixedProtocolLBService)()
+
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if tc.expectError && err != nil {
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			defer storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			if tc.expectError && err == nil {
+				t.Fatalf("unexpected success creating service")
+			}
+			createdSvc := createdObj.(*api.Service)
+			//FIXME: HACK!!  Delete above calls "inner" which doesn't
+			// yet call the allocators - no release = alloc errors!
+			defer func() {
+				for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
+					for _, ip := range createdSvc.Spec.ClusterIPs {
+						al.Release(netutils.ParseIPSloppy(ip))
+					}
+				}
+				for _, p := range createdSvc.Spec.Ports {
+					storage.alloc.serviceNodePorts.Release(int(p.NodePort))
+				}
+			}()
+
+			// Produce a map of port index to nodeport value, excluding zero.
+			ports := map[int]*api.ServicePort{}
+			for i := range createdSvc.Spec.Ports {
+				p := &createdSvc.Spec.Ports[i]
+				if p.NodePort != 0 {
+					ports[i] = p
+				}
+			}
+
+			if tc.expectNodePorts && len(ports) == 0 {
+				t.Fatalf("expected NodePorts to be allocated, found none")
+			}
+			if !tc.expectNodePorts && len(ports) > 0 {
+				t.Fatalf("expected NodePorts to not be allocated, found %v", ports)
+			}
+			if !tc.expectNodePorts {
+				return
+			}
+
+			// Make sure we got the right number of allocations
+			if want, got := len(ports), len(tc.svc.Spec.Ports); want != got {
+				t.Fatalf("expected %d NodePorts, found %d", want, got)
+			}
+
+			// Make sure they are all allocated
+			for _, p := range ports {
+				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+					t.Errorf("expected port to be allocated: %v", p)
+				}
+			}
+
+			// Make sure we got any specific allocations
+			for i, p := range tc.svc.Spec.Ports {
+				if p.NodePort != 0 {
+					if ports[i].NodePort != p.NodePort {
+						t.Errorf("expected Ports[%d].NodePort to be %d, got %d", i, p.NodePort, ports[i].NodePort)
+					}
+					// Remove requested ports from the set
+					delete(ports, i)
+				}
+			}
+
+			// Make sure any allocated ports are unique
+			seen := map[int32]int32{}
+			for i, p := range ports {
+				// We allow the same NodePort for different protocols of the
+				// same Port.
+				if prev, found := seen[p.NodePort]; found && prev != p.Port {
+					t.Errorf("found non-unique allocation in Ports[%d].NodePort: %d -> %d", i, p.NodePort, p.Port)
+				}
+				seen[p.NodePort] = p.Port
+			}
+		})
+	}
+}
+
 // Prove that a dry-run create doesn't actually allocate IPs or ports.
 func TestCreateDryRun(t *testing.T) {
 	testCases := []struct {

From a957f63ec591426a1786c41d11507251c85dea20 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 10 Jul 2021 13:59:59 -0700
Subject: [PATCH 21/79] Svc REST: HealthCheckNodePort tests

This commit ports the ExternalTrafficPolicy and HealthCheckNodePort
tests from rest_test to storage_test.  It's not a direct port, though.
I have added more cases (much more exhaustive) and more assertions.
---
 .../core/service/storage/rest_test.go         | 105 ---------
 .../core/service/storage/storage_test.go      | 219 ++++++++++++++++++
 2 files changed, 219 insertions(+), 105 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 37fc6327cae..d3e14bb9a3e 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -41,7 +41,6 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	epstest "k8s.io/kubernetes/pkg/api/endpoints/testing"
-	"k8s.io/kubernetes/pkg/api/service"
 	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
@@ -1185,110 +1184,6 @@ func TestServiceRegistryIPUpdate(t *testing.T) {
 	}
 }
 
-// Validate allocation of a nodePort when ExternalTrafficPolicy is set to Local
-// and type is LoadBalancer.
-func TestServiceRegistryCreateExternalTrafficHealthCheckNodePortAllocation(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("external-lb-esipp",
-		svctest.SetTypeLoadBalancer,
-		func(s *api.Service) {
-			s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal
-		},
-	)
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if obj == nil || err != nil {
-		t.Errorf("Unexpected failure creating service %v", err)
-	}
-
-	createdSvc := obj.(*api.Service)
-	if !service.NeedsHealthCheck(createdSvc) {
-		t.Errorf("Expecting health check needed, returned health check not needed instead")
-	}
-	port := createdSvc.Spec.HealthCheckNodePort
-	if port == 0 {
-		t.Errorf("Failed to allocate health check node port and set the HealthCheckNodePort")
-	}
-}
-
-// Validate using the user specified nodePort when ExternalTrafficPolicy is set to Local
-// and type is LoadBalancer.
-func TestServiceRegistryExternalTrafficHealthCheckNodePortUserAllocation(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("external-lb-esipp",
-		svctest.SetTypeLoadBalancer,
-		func(s *api.Service) {
-			// hard-code NodePort to make sure it doesn't conflict with the healthport.
-			// TODO: remove this once http://issue.k8s.io/93922 fixes auto-allocation conflicting with user-specified health check ports
-			s.Spec.Ports[0].NodePort = 30500
-			s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal
-			s.Spec.HealthCheckNodePort = 30501
-		},
-	)
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if obj == nil || err != nil {
-		t.Fatalf("Unexpected failure creating service :%v", err)
-	}
-
-	createdSvc := obj.(*api.Service)
-	if !service.NeedsHealthCheck(createdSvc) {
-		t.Errorf("Expecting health check needed, returned health check not needed instead")
-	}
-	port := createdSvc.Spec.HealthCheckNodePort
-	if port == 0 {
-		t.Errorf("Failed to allocate health check node port and set the HealthCheckNodePort")
-	}
-	if port != 30501 {
-		t.Errorf("Failed to allocate requested nodePort expected %d, got %d", 30501, port)
-	}
-}
-
-// Validate that the service creation fails when the requested port number is -1.
-func TestServiceRegistryCreateExternalTrafficHealthCheckNodePortNegative(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("external-lb-esipp", svctest.SetTypeLoadBalancer, func(s *api.Service) {
-		s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal
-		s.Spec.HealthCheckNodePort = int32(-1)
-	})
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if obj == nil || err != nil {
-		return
-	}
-	t.Errorf("Unexpected creation of service with invalid HealthCheckNodePort specified")
-}
-
-// Validate that the health check nodePort is not allocated when ExternalTrafficPolicy is set to Global.
-func TestServiceRegistryCreateExternalTrafficGlobal(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("external-lb-esipp",
-		svctest.SetTypeLoadBalancer,
-		func(s *api.Service) {
-			s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeCluster
-		},
-	)
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if obj == nil || err != nil {
-		t.Errorf("Unexpected failure creating service %v", err)
-	}
-
-	createdSvc := obj.(*api.Service)
-	if service.NeedsHealthCheck(createdSvc) {
-		t.Errorf("Expecting health check not needed, returned health check needed instead")
-	}
-	// Make sure the service does not have the health check node port allocated
-	port := createdSvc.Spec.HealthCheckNodePort
-	if port != 0 {
-		t.Errorf("Unexpected allocation of health check node port: %v", port)
-	}
-}
-
 // Validate the internalTrafficPolicy field when set to "Cluster" then updated to "Local"
 func TestServiceRegistryInternalTrafficPolicyClusterThenLocal(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 52cfc42e210..c5933c30d7e 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5126,6 +5126,225 @@ func TestCreateInitNodePorts(t *testing.T) {
 	}
 }
 
+func TestCreateExternalTrafficPolicy(t *testing.T) {
+	testCases := []struct {
+		name        string
+		svc         *api.Service
+		expectError bool
+		expectHCNP  bool
+	}{{
+		name: "ExternalName_policy:none_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy("")),
+		expectHCNP: false,
+	}, {
+		name: "ExternalName_policy:none_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy(""),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "ExternalName_policy:Cluster_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+		expectError: true,
+	}, {
+		name: "ExternalName_policy:Cluster_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "ExternalName_policy:Local_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		expectError: true,
+	}, {
+		name: "ExternalName_policy:Local_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "ClusterIP_policy:none_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetExternalTrafficPolicy("")),
+		expectHCNP: false,
+	}, {
+		name: "ClusterIP_policy:none_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetExternalTrafficPolicy(""),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "ClusterIP_policy:Cluster_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+		expectError: true,
+	}, {
+		name: "ClusterIP_policy:Cluster_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "ClusterIP_policy:Local_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		expectError: true,
+	}, {
+		name: "ClusterIP_policy:Local_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "NodePort_policy:none_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetExternalTrafficPolicy("")),
+		expectHCNP: false,
+	}, {
+		name: "NodePort_policy:none_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetExternalTrafficPolicy(""),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "NodePort_policy:Cluster:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+		expectHCNP: false,
+	}, {
+		name: "NodePort_policy:Cluster:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "NodePort_policy:Local_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		expectHCNP: false,
+	}, {
+		name: "NodePort_policy:Local_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "LoadBalancer_policy:none_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy("")),
+		expectHCNP: false,
+	}, {
+		name: "LoadBalancer_policy:none_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(""),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "LoadBalancer_policy:Cluster_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+		expectHCNP: false,
+	}, {
+		name: "LoadBalancer_policy:Cluster_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectError: true,
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:none",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		expectHCNP: true,
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:specified",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(30000)),
+		expectHCNP: true,
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:negative",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(-1)),
+		expectError: true,
+	}}
+
+	// Do this in the outer scope for performance.
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if tc.expectError && err != nil {
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			defer storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			if tc.expectError && err == nil {
+				t.Fatalf("unexpected success creating service")
+			}
+			createdSvc := createdObj.(*api.Service)
+
+			if !tc.expectHCNP {
+				if createdSvc.Spec.HealthCheckNodePort != 0 {
+					t.Fatalf("expected no HealthCheckNodePort, got %d", createdSvc.Spec.HealthCheckNodePort)
+				}
+				return
+			}
+
+			if createdSvc.Spec.HealthCheckNodePort == 0 {
+				t.Fatalf("expected a HealthCheckNodePort")
+			}
+			if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+				t.Errorf("expected HealthCheckNodePort to be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
+			}
+			if tc.svc.Spec.HealthCheckNodePort != 0 {
+				if want, got := tc.svc.Spec.HealthCheckNodePort, createdSvc.Spec.HealthCheckNodePort; want != got {
+					t.Errorf("wrong HealthCheckNodePort value: wanted %d, got %d", want, got)
+				}
+			}
+			for i, p := range createdSvc.Spec.Ports {
+				if p.NodePort == createdSvc.Spec.HealthCheckNodePort {
+					t.Errorf("HealthCheckNodePort overlaps NodePort[%d]", i)
+				}
+			}
+		})
+	}
+}
+
 // Prove that a dry-run create doesn't actually allocate IPs or ports.
 func TestCreateDryRun(t *testing.T) {
 	testCases := []struct {

From 15c513cc363164fd1bbe51c9ad0fa65fe18ae1d5 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 10 Jul 2021 18:54:04 -0700
Subject: [PATCH 22/79] Svc REST: IP and port reallocation

Make sure the logic that was covered in rest_test is covered in
storage_test.
---
 .../core/service/storage/rest_test.go         | 91 -------------------
 .../core/service/storage/storage_test.go      | 79 ++++++++++++++--
 2 files changed, 71 insertions(+), 99 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index d3e14bb9a3e..33850c2ab23 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -1043,97 +1043,6 @@ func TestServiceRegistryList(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryCreateIPAllocation(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	svc1 := svctest.MakeService("foo")
-	ctx := genericapirequest.NewDefaultContext()
-	obj, err := storage.Create(ctx, svc1, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("error creating service: %v", err)
-	}
-	createdSvc1 := obj.(*api.Service)
-	if createdSvc1.Name != "foo" {
-		t.Errorf("Expected foo, but got %v", createdSvc1.Name)
-	}
-	if !makeIPNet(t).Contains(netutils.ParseIPSloppy(createdSvc1.Spec.ClusterIPs[0])) {
-		t.Errorf("Unexpected ClusterIP: %s", createdSvc1.Spec.ClusterIPs[0])
-	}
-
-	svc2 := svctest.MakeService("bar")
-	ctx = genericapirequest.NewDefaultContext()
-	obj, err = storage.Create(ctx, svc2, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("error creating service: %v", err)
-	}
-	createdSvc2 := obj.(*api.Service)
-	if createdSvc2.Name != "bar" {
-		t.Errorf("Expected bar, but got %v", createdSvc2.Name)
-	}
-	if !makeIPNet(t).Contains(netutils.ParseIPSloppy(createdSvc2.Spec.ClusterIPs[0])) {
-		t.Errorf("Unexpected ClusterIP: %s", createdSvc2.Spec.ClusterIPs[0])
-	}
-
-	testIPs := []string{"1.2.3.93", "1.2.3.94", "1.2.3.95", "1.2.3.96"}
-	testIP := "not-an-ip"
-	for _, ip := range testIPs {
-		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily].(*ipallocator.Range), ip) {
-			testIP = ip
-			break
-		}
-	}
-
-	svc3 := svctest.MakeService("qux", svctest.SetClusterIPs(testIP))
-	ctx = genericapirequest.NewDefaultContext()
-	obj, err = storage.Create(ctx, svc3, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	createdSvc3 := obj.(*api.Service)
-	if createdSvc3.Spec.ClusterIPs[0] != testIP { // specific IP
-		t.Errorf("Unexpected ClusterIP: %s", createdSvc3.Spec.ClusterIPs[0])
-	}
-}
-
-func TestServiceRegistryCreateIPReallocation(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	svc1 := svctest.MakeService("foo")
-	ctx := genericapirequest.NewDefaultContext()
-	obj, err := storage.Create(ctx, svc1, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("error creating service: %v", err)
-	}
-	createdSvc1 := obj.(*api.Service)
-	if createdSvc1.Name != "foo" {
-		t.Errorf("Expected foo, but got %v", createdSvc1.Name)
-	}
-	if !makeIPNet(t).Contains(netutils.ParseIPSloppy(createdSvc1.Spec.ClusterIPs[0])) {
-		t.Errorf("Unexpected ClusterIP: %s", createdSvc1.Spec.ClusterIPs[0])
-	}
-
-	_, _, err = storage.Delete(ctx, createdSvc1.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
-	if err != nil {
-		t.Errorf("Unexpected error deleting service: %v", err)
-	}
-
-	svc2 := svctest.MakeService("bar", svctest.SetClusterIPs(createdSvc1.Spec.ClusterIP))
-	ctx = genericapirequest.NewDefaultContext()
-	obj, err = storage.Create(ctx, svc2, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("error creating service: %v", err)
-	}
-	createdSvc2 := obj.(*api.Service)
-	if createdSvc2.Name != "bar" {
-		t.Errorf("Expected bar, but got %v", createdSvc2.Name)
-	}
-	if !makeIPNet(t).Contains(netutils.ParseIPSloppy(createdSvc2.Spec.ClusterIPs[0])) {
-		t.Errorf("Unexpected ClusterIP: %s", createdSvc2.Spec.ClusterIPs[0])
-	}
-}
-
 func TestServiceRegistryIPUpdate(t *testing.T) {
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
 	defer server.Terminate(t)
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index c5933c30d7e..828742cb60f 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -4715,16 +4715,19 @@ func TestCreateInitIPFields(t *testing.T) {
 					}
 					if itc.expectHeadless {
 						if !reflect.DeepEqual(createdSvc.Spec.ClusterIPs, []string{"None"}) {
-							t.Errorf("wrong clusterIPs: want [\"None\"], got %v", createdSvc.Spec.ClusterIPs)
+							t.Fatalf("wrong clusterIPs: want [\"None\"], got %v", createdSvc.Spec.ClusterIPs)
 						}
-					} else {
-						if c, f := len(createdSvc.Spec.ClusterIPs), len(createdSvc.Spec.IPFamilies); c != f {
-							t.Errorf("clusterIPs and ipFamilies are not the same length: %d vs %d", c, f)
+						return
+					}
+					if c, f := len(createdSvc.Spec.ClusterIPs), len(createdSvc.Spec.IPFamilies); c != f {
+						t.Errorf("clusterIPs and ipFamilies are not the same length: %d vs %d", c, f)
+					}
+					for i, clip := range createdSvc.Spec.ClusterIPs {
+						if cf, ef := familyOf(clip), createdSvc.Spec.IPFamilies[i]; cf != ef {
+							t.Errorf("clusterIP is the wrong IP family: want %s, got %s", ef, cf)
 						}
-						for i, clip := range createdSvc.Spec.ClusterIPs {
-							if cf, ef := familyOf(clip), createdSvc.Spec.IPFamilies[i]; cf != ef {
-								t.Errorf("clusterIP is the wrong IP family: want %s, got %s", ef, cf)
-							}
+						if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
+							t.Errorf("clusterIP is not allocated: %v", clip)
 						}
 					}
 				})
@@ -4733,6 +4736,66 @@ func TestCreateInitIPFields(t *testing.T) {
 	}
 }
 
+func TestCreateReallocation(t *testing.T) {
+	testCases := []struct {
+		name string
+		svc  *api.Service
+	}{{
+		name: "v4",
+		svc:  svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetIPFamilies(api.IPv4Protocol)),
+	}, {
+		name: "v6",
+		svc:  svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetIPFamilies(api.IPv6Protocol)),
+	}, {
+		name: "v4v6",
+		svc:  svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+	}}
+
+	// This test is ONLY with the gate enabled.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			createdSvc := createdObj.(*api.Service)
+
+			_, _, err = storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			//FIXME: HACK!!  Delete above calls "inner" which doesn't
+			// yet call the allocators - no release = test errors!
+			for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
+				for _, ip := range createdSvc.Spec.ClusterIPs {
+					al.Release(netutils.ParseIPSloppy(ip))
+				}
+			}
+			for _, p := range createdSvc.Spec.Ports {
+				storage.alloc.serviceNodePorts.Release(int(p.NodePort))
+			}
+
+			// Force the same IPs and ports
+			svc2 := tc.svc.DeepCopy()
+			svc2.Spec.ClusterIP = createdSvc.Spec.ClusterIP
+			svc2.Spec.ClusterIPs = createdSvc.Spec.ClusterIPs
+			svc2.Spec.Ports = createdSvc.Spec.Ports
+
+			_, err = storage.Create(ctx, svc2, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+		})
+	}
+}
+
 func TestCreateInitNodePorts(t *testing.T) {
 	testCases := []struct {
 		name                         string

From 6d640aa244136e9b4dcdb9f8e49b459ee72536d0 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 11 Jul 2021 11:14:16 -0700
Subject: [PATCH 23/79] Svc REST: Remove redundant Get test

---
 pkg/registry/core/service/storage/rest_test.go | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 33850c2ab23..8bd27fe8144 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -848,21 +848,6 @@ func TestServiceRegistryUpdateMultiPortLoadBalancerService(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryGet(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	_, err := storage.Create(ctx, svctest.MakeService("foo"), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("error creating service: %v", err)
-	}
-	obj, _ := storage.Get(ctx, "foo", &metav1.GetOptions{})
-	svc := obj.(*api.Service)
-	if e, a := "foo", svc.Name; e != a {
-		t.Errorf("Expected %v, but got %v", e, a)
-	}
-}
-
 // this is local because it's not fully fleshed out enough for general use.
 func makePod(name string, ips ...string) api.Pod {
 	p := api.Pod{

From 42b53d850d463934fb89935a2363003e3f297862 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Thu, 8 Jul 2021 17:07:55 -0700
Subject: [PATCH 24/79] Svc REST: Move test to reduce diff in next commits

No changes - just move.
---
 .../core/service/storage/storage_test.go      | 152 +++++++++---------
 1 file changed, 76 insertions(+), 76 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 828742cb60f..0d1ce05e1c3 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5408,6 +5408,82 @@ func TestCreateExternalTrafficPolicy(t *testing.T) {
 	}
 }
 
+// Prove that create skips allocations for Headless services.
+func TestCreateSkipsAllocationsForHeadless(t *testing.T) {
+	testCases := []struct {
+		name            string
+		clusterFamilies []api.IPFamily
+		enableDualStack bool
+		svc             *api.Service
+		expectError     bool
+	}{{
+		name:            "singlestack:v4_gate:off",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v6_gate:on",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "dualstack:v4v6_gate:off",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "dualstack:v6v4_gate:on",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo"),
+	}, {
+		name:            "singlestack:v4_gate:off_type:NodePort",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		enableDualStack: false,
+		svc:             svctest.MakeService("foo", svctest.SetTypeNodePort),
+		expectError:     true,
+	}, {
+		name:            "singlestack:v6_gate:on_type:LoadBalancer",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		enableDualStack: true,
+		svc:             svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+		expectError:     true,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
+
+			storage, _, server := newStorage(t, tc.clusterFamilies)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			// This test is ONLY headless services.
+			tc.svc.Spec.ClusterIP = api.ClusterIPNone
+
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if tc.expectError && err != nil {
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			if tc.expectError && err == nil {
+				t.Fatalf("unexpected success creating service")
+			}
+			createdSvc := createdObj.(*api.Service)
+
+			if createdSvc.Spec.ClusterIP != "None" {
+				t.Errorf("expected clusterIP \"None\", got %q", createdSvc.Spec.ClusterIP)
+			}
+			if !reflect.DeepEqual(createdSvc.Spec.ClusterIPs, []string{"None"}) {
+				t.Errorf("expected clusterIPs [\"None\"], got %q", createdSvc.Spec.ClusterIPs)
+			}
+		})
+	}
+}
+
 // Prove that a dry-run create doesn't actually allocate IPs or ports.
 func TestCreateDryRun(t *testing.T) {
 	testCases := []struct {
@@ -5505,79 +5581,3 @@ func TestCreateDryRun(t *testing.T) {
 		})
 	}
 }
-
-// Prove that create skips allocations for Headless services.
-func TestCreateSkipsAllocationsForHeadless(t *testing.T) {
-	testCases := []struct {
-		name            string
-		clusterFamilies []api.IPFamily
-		enableDualStack bool
-		svc             *api.Service
-		expectError     bool
-	}{{
-		name:            "singlestack:v4_gate:off_type:ClusterIP",
-		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
-		enableDualStack: false,
-		svc:             svctest.MakeService("foo"),
-	}, {
-		name:            "singlestack:v6_gate:on_type:ClusterIP",
-		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
-		enableDualStack: true,
-		svc:             svctest.MakeService("foo"),
-	}, {
-		name:            "dualstack:v4v6_gate:off_type:ClusterIP",
-		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-		enableDualStack: false,
-		svc:             svctest.MakeService("foo"),
-	}, {
-		name:            "dualstack:v6v4_gate:on_type:ClusterIP",
-		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-		enableDualStack: true,
-		svc:             svctest.MakeService("foo"),
-	}, {
-		name:            "singlestack:v4_gate:off_type:NodePort",
-		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
-		enableDualStack: false,
-		svc:             svctest.MakeService("foo", svctest.SetTypeNodePort),
-		expectError:     true,
-	}, {
-		name:            "singlestack:v6_gate:on_type:LoadBalancer",
-		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
-		enableDualStack: true,
-		svc:             svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
-		expectError:     true,
-	}}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
-
-			storage, _, server := newStorage(t, tc.clusterFamilies)
-			defer server.Terminate(t)
-			defer storage.Store.DestroyFunc()
-
-			// This test is ONLY headless services.
-			tc.svc.Spec.ClusterIP = api.ClusterIPNone
-
-			ctx := genericapirequest.NewDefaultContext()
-			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if tc.expectError && err != nil {
-				return
-			}
-			if err != nil {
-				t.Fatalf("unexpected error creating service: %v", err)
-			}
-			if tc.expectError && err == nil {
-				t.Fatalf("unexpected success creating service")
-			}
-			createdSvc := createdObj.(*api.Service)
-
-			if createdSvc.Spec.ClusterIP != "None" {
-				t.Errorf("expected clusterIP \"None\", got %q", createdSvc.Spec.ClusterIP)
-			}
-			if !reflect.DeepEqual(createdSvc.Spec.ClusterIPs, []string{"None"}) {
-				t.Errorf("expected clusterIPs [\"None\"], got %q", createdSvc.Spec.ClusterIPs)
-			}
-		})
-	}
-}

From 61a5e7498d0c188c74f5133e511799e358739337 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 27 Nov 2020 17:19:28 -0800
Subject: [PATCH 25/79] Svc REST: De-layer Delete

Gut the "outer" Delete() and move it to the inner AfterDelete().
---
 api/openapi-spec/swagger.json                 | 125 ++++++++++++-
 pkg/registry/core/rest/storage_core.go        |   7 +-
 pkg/registry/core/service/storage/rest.go     |  49 +----
 .../core/service/storage/rest_test.go         |  20 +-
 pkg/registry/core/service/storage/storage.go  |  47 ++++-
 .../core/service/storage/storage_test.go      | 177 +++++++++++++-----
 6 files changed, 326 insertions(+), 99 deletions(-)

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
index b53b14f1e15..112d2456bf7 100644
--- a/api/openapi-spec/swagger.json
+++ b/api/openapi-spec/swagger.json
@@ -25973,6 +25973,127 @@
       }
     },
     "/api/v1/namespaces/{namespace}/services": {
+      "delete": {
+        "consumes": [
+          "*/*"
+        ],
+        "description": "delete collection of Service",
+        "operationId": "deleteCoreV1CollectionNamespacedService",
+        "parameters": [
+          {
+            "in": "body",
+            "name": "body",
+            "schema": {
+              "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.DeleteOptions"
+            }
+          },
+          {
+            "description": "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.",
+            "in": "query",
+            "name": "continue",
+            "type": "string",
+            "uniqueItems": true
+          },
+          {
+            "description": "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed",
+            "in": "query",
+            "name": "dryRun",
+            "type": "string",
+            "uniqueItems": true
+          },
+          {
+            "description": "A selector to restrict the list of returned objects by their fields. Defaults to everything.",
+            "in": "query",
+            "name": "fieldSelector",
+            "type": "string",
+            "uniqueItems": true
+          },
+          {
+            "description": "The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.",
+            "in": "query",
+            "name": "gracePeriodSeconds",
+            "type": "integer",
+            "uniqueItems": true
+          },
+          {
+            "description": "A selector to restrict the list of returned objects by their labels. Defaults to everything.",
+            "in": "query",
+            "name": "labelSelector",
+            "type": "string",
+            "uniqueItems": true
+          },
+          {
+            "description": "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.",
+            "in": "query",
+            "name": "limit",
+            "type": "integer",
+            "uniqueItems": true
+          },
+          {
+            "description": "Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the \"orphan\" finalizer will be added to/removed from the object's finalizers list. Either this field or PropagationPolicy may be set, but not both.",
+            "in": "query",
+            "name": "orphanDependents",
+            "type": "boolean",
+            "uniqueItems": true
+          },
+          {
+            "description": "Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: 'Orphan' - orphan the dependents; 'Background' - allow the garbage collector to delete the dependents in the background; 'Foreground' - a cascading policy that deletes all dependents in the foreground.",
+            "in": "query",
+            "name": "propagationPolicy",
+            "type": "string",
+            "uniqueItems": true
+          },
+          {
+            "description": "resourceVersion sets a constraint on what resource versions a request may be served from. See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.\n\nDefaults to unset",
+            "in": "query",
+            "name": "resourceVersion",
+            "type": "string",
+            "uniqueItems": true
+          },
+          {
+            "description": "resourceVersionMatch determines how resourceVersion is applied to list calls. It is highly recommended that resourceVersionMatch be set for list calls where resourceVersion is set See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.\n\nDefaults to unset",
+            "in": "query",
+            "name": "resourceVersionMatch",
+            "type": "string",
+            "uniqueItems": true
+          },
+          {
+            "description": "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.",
+            "in": "query",
+            "name": "timeoutSeconds",
+            "type": "integer",
+            "uniqueItems": true
+          }
+        ],
+        "produces": [
+          "application/json",
+          "application/yaml",
+          "application/vnd.kubernetes.protobuf"
+        ],
+        "responses": {
+          "200": {
+            "description": "OK",
+            "schema": {
+              "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Status"
+            }
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        },
+        "schemes": [
+          "https"
+        ],
+        "tags": [
+          "core_v1"
+        ],
+        "x-kubernetes-action": "deletecollection",
+        "x-kubernetes-group-version-kind": {
+          "group": "",
+          "kind": "Service",
+          "version": "v1"
+        }
+      },
       "get": {
         "consumes": [
           "*/*"
@@ -26217,13 +26338,13 @@
           "200": {
             "description": "OK",
             "schema": {
-              "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Status"
+              "$ref": "#/definitions/io.k8s.api.core.v1.Service"
             }
           },
           "202": {
             "description": "Accepted",
             "schema": {
-              "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Status"
+              "$ref": "#/definitions/io.k8s.api.core.v1.Service"
             }
           },
           "401": {
diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go
index d2fa6a1d45f..61af3b7c0e8 100644
--- a/pkg/registry/core/rest/storage_core.go
+++ b/pkg/registry/core/rest/storage_core.go
@@ -261,7 +261,12 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		serviceIPAllocators[secondaryServiceClusterIPAllocator.IPFamily()] = secondaryServiceClusterIPAllocator
 	}
 
-	serviceRESTStorage, serviceStatusStorage, err := servicestore.NewGenericREST(restOptionsGetter, serviceClusterIPAllocator.IPFamily(), serviceIPAllocators, serviceNodePortAllocator)
+	serviceRESTStorage, serviceStatusStorage, err := servicestore.NewGenericREST(
+		restOptionsGetter,
+		serviceClusterIPAllocator.IPFamily(),
+		serviceIPAllocators,
+		serviceNodePortAllocator,
+		endpointsStorage)
 	if err != nil {
 		return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err
 	}
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 1f3d07dd805..06304661746 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -87,11 +87,6 @@ type ServiceStorage interface {
 	rest.ResetFieldsStrategy
 }
 
-type EndpointsStorage interface {
-	rest.Getter
-	rest.GracefulDeleter
-}
-
 // NewREST returns a wrapper around the underlying generic storage and performs
 // allocations and deallocations of various service related resources like ports.
 // TODO: all transactional behavior should be supported from within generic storage
@@ -212,49 +207,7 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 }
 
 func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
-	// TODO: handle graceful
-	obj, _, err := rs.services.Delete(ctx, id, deleteValidation, options)
-	if err != nil {
-		return nil, false, err
-	}
-
-	svc := obj.(*api.Service)
-	// (khenidak) double check that this is in fact the best place for this
-
-	// delete strategy handles graceful delete only. It expects strategy
-	// to implement Graceful-Delete related interface. Hence we are not doing
-	// the below there. instead we are doing it locally. Until strategy.BeforeDelete works without
-	// having to implement graceful delete management
-	// set ClusterIPs based on ClusterIP
-	// because we depend on ClusterIPs and data might be saved without ClusterIPs ..
-
-	if svc.Spec.ClusterIPs == nil && len(svc.Spec.ClusterIP) > 0 {
-		svc.Spec.ClusterIPs = []string{svc.Spec.ClusterIP}
-	}
-
-	// Only perform the cleanup if this is a non-dryrun deletion
-	if !dryrun.IsDryRun(options.DryRun) {
-		// TODO: can leave dangling endpoints, and potentially return incorrect
-		// endpoints if a new service is created with the same name
-		_, _, err = rs.endpoints.Delete(ctx, id, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
-		if err != nil && !errors.IsNotFound(err) {
-			return nil, false, err
-		}
-
-		rs.alloc.releaseAllocatedResources(svc)
-	}
-
-	// TODO: this is duplicated from the generic storage, when this wrapper is fully removed we can drop this
-	details := &metav1.StatusDetails{
-		Name: svc.Name,
-		UID:  svc.UID,
-	}
-	if info, ok := genericapirequest.RequestInfoFrom(ctx); ok {
-		details.Group = info.APIGroup
-		details.Kind = info.Resource // legacy behavior
-	}
-	status := &metav1.Status{Status: metav1.StatusSuccess, Details: details}
-	return status, true, nil
+	return rs.services.Delete(ctx, id, deleteValidation, options)
 }
 
 func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 8bd27fe8144..0e805bc3df2 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -144,9 +144,17 @@ func (s *serviceStorage) Update(ctx context.Context, name string, objInfo rest.U
 }
 
 func (s *serviceStorage) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
-	ret := s.Services[name]
+	ret, del, err := s.inner.Delete(ctx, name, deleteValidation, options)
+	if err != nil {
+		return ret, del, err
+	}
+
+	if dryrun.IsDryRun(options.DryRun) {
+		return ret.DeepCopyObject(), del, nil
+	}
 	delete(s.Services, name)
-	return ret, false, nil
+
+	return ret.DeepCopyObject(), del, err
 }
 
 func (s *serviceStorage) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) {
@@ -264,7 +272,13 @@ func newInnerREST(t *testing.T, etcdStorage *storagebackend.ConfigForResource, i
 		DeleteCollectionWorkers: 1,
 		ResourcePrefix:          "services",
 	}
-	inner, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, portAlloc)
+	endpoints, err := endpointstore.NewREST(generic.RESTOptions{
+		StorageConfig:  etcdStorage,
+		Decorator:      generic.UndecoratedStorage,
+		ResourcePrefix: "endpoints",
+	})
+
+	inner, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, portAlloc, endpoints)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 6b0e1a81309..6e18ad27d97 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -21,11 +21,14 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/apiserver/pkg/util/dryrun"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/cri-api/pkg/errors"
+	"k8s.io/klog/v2"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/printers"
@@ -40,11 +43,17 @@ import (
 	netutil "k8s.io/utils/net"
 )
 
+type EndpointsStorage interface {
+	rest.Getter
+	rest.GracefulDeleter
+}
+
 type GenericREST struct {
 	*genericregistry.Store
 	primaryIPFamily   api.IPFamily
 	secondaryIPFamily api.IPFamily
 	alloc             RESTAllocStuff
+	endpoints         EndpointsStorage
 }
 
 // NewGenericREST returns a RESTStorage object that will work against services.
@@ -52,7 +61,8 @@ func NewGenericREST(
 	optsGetter generic.RESTOptionsGetter,
 	serviceIPFamily api.IPFamily,
 	ipAllocs map[api.IPFamily]ipallocator.Interface,
-	portAlloc portallocator.Interface) (*GenericREST, *StatusREST, error) {
+	portAlloc portallocator.Interface,
+	endpoints EndpointsStorage) (*GenericREST, *StatusREST, error) {
 
 	strategy, _ := svcreg.StrategyForServiceCIDRs(ipAllocs[serviceIPFamily].CIDR(), len(ipAllocs) > 1)
 
@@ -84,8 +94,15 @@ func NewGenericREST(
 	if len(ipAllocs) > 1 {
 		secondaryIPFamily = otherFamily(serviceIPFamily)
 	}
-	genericStore := &GenericREST{store, primaryIPFamily, secondaryIPFamily, makeAlloc(serviceIPFamily, ipAllocs, portAlloc)}
+	genericStore := &GenericREST{
+		Store:             store,
+		primaryIPFamily:   primaryIPFamily,
+		secondaryIPFamily: secondaryIPFamily,
+		alloc:             makeAlloc(serviceIPFamily, ipAllocs, portAlloc),
+		endpoints:         endpoints,
+	}
 	store.Decorator = genericStore.defaultOnRead
+	store.AfterDelete = genericStore.afterDelete
 	store.BeginCreate = genericStore.beginCreate
 	store.BeginUpdate = genericStore.beginUpdate
 
@@ -249,6 +266,32 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 	}
 }
 
+func (r *GenericREST) afterDelete(obj runtime.Object, options *metav1.DeleteOptions) {
+	svc := obj.(*api.Service)
+
+	// Normally this defaulting is done automatically, but the hook (Decorator)
+	// is called at the end of this process, and we want the fully-formed
+	// object.
+	r.defaultOnReadService(svc)
+
+	// Only perform the cleanup if this is a non-dryrun deletion
+	if !dryrun.IsDryRun(options.DryRun) {
+		// It would be better if we had the caller context, but that changes
+		// this hook signature.
+		ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), svc.Namespace)
+		// TODO: This is clumsy.  It was added for fear that the endpoints
+		// controller might lag, and we could end up rusing the service name
+		// with old endpoints.  We should solve that better and remove this, or
+		// else we should do this for EndpointSlice, too.
+		_, _, err := r.endpoints.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+		if err != nil && !errors.IsNotFound(err) {
+			klog.Errorf("delete service endpoints %s/%s failed: %v", svc.Name, svc.Namespace, err)
+		}
+
+		r.alloc.releaseAllocatedResources(svc)
+	}
+}
+
 func (r *GenericREST) beginCreate(ctx context.Context, obj runtime.Object, options *metav1.CreateOptions) (genericregistry.FinishFunc, error) {
 	svc := obj.(*api.Service)
 
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 0d1ce05e1c3..88167e379b0 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package storage
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"reflect"
@@ -61,6 +62,16 @@ func makePortAllocator(ports machineryutilnet.PortRange) portallocator.Interface
 	return al
 }
 
+type fakeEndpoints struct{}
+
+func (fakeEndpoints) Delete(_ context.Context, _ string, _ rest.ValidateObjectFunc, _ *metav1.DeleteOptions) (runtime.Object, bool, error) {
+	return nil, false, nil
+}
+
+func (fakeEndpoints) Get(_ context.Context, _ string, _ *metav1.GetOptions) (runtime.Object, error) {
+	return nil, nil
+}
+
 func ipIsAllocated(t *testing.T, alloc ipallocator.Interface, ipstr string) bool {
 	t.Helper()
 	ip := netutils.ParseIPSloppy(ipstr)
@@ -105,7 +116,7 @@ func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusR
 
 	portAlloc := makePortAllocator(*(machineryutilnet.ParsePortRangeOrDie("30000-32767")))
 
-	serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, portAlloc)
+	serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, portAlloc, fakeEndpoints{})
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
@@ -640,15 +651,6 @@ func TestCreateIgnoresIPFamilyWithoutDualStack(t *testing.T) {
 			}
 			defer storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
 			createdSvc := createdObj.(*api.Service)
-			//FIXME: HACK!!  Delete above calls "inner" which doesn't
-			// yet call the allocators - no release = alloc errors!
-			defer func() {
-				for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
-					for _, ip := range createdSvc.Spec.ClusterIPs {
-						al.Release(netutils.ParseIPSloppy(ip))
-					}
-				}
-			}()
 
 			// The gate is off - these should always be empty.
 			if want, got := fmtIPFamilyPolicy(nil), fmtIPFamilyPolicy(createdSvc.Spec.IPFamilyPolicy); want != got {
@@ -4697,15 +4699,6 @@ func TestCreateInitIPFields(t *testing.T) {
 						t.Fatalf("unexpected success creating service")
 					}
 					createdSvc := createdObj.(*api.Service)
-					//FIXME: HACK!!  Delete above calls "inner" which doesn't
-					// yet call the allocators - no release = alloc errors!
-					defer func() {
-						for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
-							for _, ip := range createdSvc.Spec.ClusterIPs {
-								al.Release(netutils.ParseIPSloppy(ip))
-							}
-						}
-					}()
 
 					if want, got := fmtIPFamilyPolicy(&itc.expectPolicy), fmtIPFamilyPolicy(createdSvc.Spec.IPFamilyPolicy); want != got {
 						t.Errorf("wrong IPFamilyPolicy: want %s, got %s", want, got)
@@ -4736,7 +4729,7 @@ func TestCreateInitIPFields(t *testing.T) {
 	}
 }
 
-func TestCreateReallocation(t *testing.T) {
+func TestCreateDeleteReuse(t *testing.T) {
 	testCases := []struct {
 		name string
 		svc  *api.Service
@@ -4761,37 +4754,69 @@ func TestCreateReallocation(t *testing.T) {
 			defer storage.Store.DestroyFunc()
 
 			ctx := genericapirequest.NewDefaultContext()
+
+			// Create it
 			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
 			if err != nil {
 				t.Fatalf("unexpected error creating service: %v", err)
 			}
 			createdSvc := createdObj.(*api.Service)
 
+			// Ensure IPs and ports were allocated
+			for i, fam := range createdSvc.Spec.IPFamilies {
+				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+					t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
+				}
+			}
+			for _, p := range createdSvc.Spec.Ports {
+				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+					t.Errorf("expected port to be allocated: %v", p.NodePort)
+				}
+			}
+
+			// Delete it
 			_, _, err = storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
 			if err != nil {
 				t.Fatalf("unexpected error creating service: %v", err)
 			}
-			//FIXME: HACK!!  Delete above calls "inner" which doesn't
-			// yet call the allocators - no release = test errors!
-			for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
-				for _, ip := range createdSvc.Spec.ClusterIPs {
-					al.Release(netutils.ParseIPSloppy(ip))
+
+			// Ensure IPs and ports were deallocated
+			for i, fam := range createdSvc.Spec.IPFamilies {
+				if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+					t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIPs[i])
 				}
 			}
 			for _, p := range createdSvc.Spec.Ports {
-				storage.alloc.serviceNodePorts.Release(int(p.NodePort))
+				if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+					t.Errorf("expected port to not be allocated: %v", p.NodePort)
+				}
 			}
 
 			// Force the same IPs and ports
 			svc2 := tc.svc.DeepCopy()
+			svc2.Name += "2"
 			svc2.Spec.ClusterIP = createdSvc.Spec.ClusterIP
 			svc2.Spec.ClusterIPs = createdSvc.Spec.ClusterIPs
 			svc2.Spec.Ports = createdSvc.Spec.Ports
 
+			// Create again
 			_, err = storage.Create(ctx, svc2, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
 			if err != nil {
 				t.Fatalf("unexpected error creating service: %v", err)
 			}
+
+			// Ensure IPs and ports were allocated
+			for i, fam := range createdSvc.Spec.IPFamilies {
+				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+					t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
+				}
+			}
+			for _, p := range createdSvc.Spec.Ports {
+				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+					t.Errorf("expected port to be allocated: %v", p.NodePort)
+				}
+			}
+
 		})
 	}
 }
@@ -5120,18 +5145,6 @@ func TestCreateInitNodePorts(t *testing.T) {
 				t.Fatalf("unexpected success creating service")
 			}
 			createdSvc := createdObj.(*api.Service)
-			//FIXME: HACK!!  Delete above calls "inner" which doesn't
-			// yet call the allocators - no release = alloc errors!
-			defer func() {
-				for _, al := range storage.alloc.serviceIPAllocatorsByFamily {
-					for _, ip := range createdSvc.Spec.ClusterIPs {
-						al.Release(netutils.ParseIPSloppy(ip))
-					}
-				}
-				for _, p := range createdSvc.Spec.Ports {
-					storage.alloc.serviceNodePorts.Release(int(p.NodePort))
-				}
-			}()
 
 			// Produce a map of port index to nodeport value, excluding zero.
 			ports := map[int]*api.ServicePort{}
@@ -5550,23 +5563,23 @@ func TestCreateDryRun(t *testing.T) {
 
 			// Ensure IPs were allocated
 			if netutils.ParseIPSloppy(createdSvc.Spec.ClusterIP) == nil {
-				t.Errorf("expected valid clusterIP: %v", createdSvc.Spec.ClusterIP)
+				t.Errorf("expected valid clusterIP: %q", createdSvc.Spec.ClusterIP)
 			}
 
 			// Ensure the IP allocators are clean.
 			if !tc.enableDualStack {
 				if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol], createdSvc.Spec.ClusterIP) {
-					t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIP)
+					t.Errorf("expected IP to not be allocated: %q", createdSvc.Spec.ClusterIP)
 				}
 			} else {
 				for _, ip := range createdSvc.Spec.ClusterIPs {
 					if netutils.ParseIPSloppy(ip) == nil {
-						t.Errorf("expected valid clusterIP: %v", createdSvc.Spec.ClusterIP)
+						t.Errorf("expected valid clusterIP: %q", createdSvc.Spec.ClusterIP)
 					}
 				}
 				for i, fam := range createdSvc.Spec.IPFamilies {
 					if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-						t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIPs[i])
+						t.Errorf("expected IP to not be allocated: %q", createdSvc.Spec.ClusterIPs[i])
 					}
 				}
 			}
@@ -5574,10 +5587,88 @@ func TestCreateDryRun(t *testing.T) {
 			if tc.svc.Spec.Type != api.ServiceTypeClusterIP {
 				for _, p := range createdSvc.Spec.Ports {
 					if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-						t.Errorf("expected port to not be allocated: %v", p.NodePort)
+						t.Errorf("expected port to not be allocated: %d", p.NodePort)
 					}
 				}
 			}
 		})
 	}
 }
+
+// Prove that a dry-run delete doesn't actually deallocate IPs or ports.
+func TestDeleteDryRun(t *testing.T) {
+	testCases := []struct {
+		name            string
+		enableDualStack bool
+		svc             *api.Service
+	}{{
+		name:            "gate:off",
+		enableDualStack: false,
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+	}, {
+		name:            "gate:on",
+		enableDualStack: true,
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
+
+			families := []api.IPFamily{api.IPv4Protocol}
+			if tc.enableDualStack {
+				families = append(families, api.IPv6Protocol)
+			}
+
+			storage, _, server := newStorage(t, families)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			ctx := genericapirequest.NewDefaultContext()
+			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			createdSvc := createdObj.(*api.Service)
+
+			// Ensure IPs and ports were allocated
+			for i, fam := range createdSvc.Spec.IPFamilies {
+				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+					t.Errorf("expected IP to be allocated: %q", createdSvc.Spec.ClusterIPs[i])
+				}
+			}
+			for _, p := range createdSvc.Spec.Ports {
+				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+					t.Errorf("expected port to be allocated: %d", p.NodePort)
+				}
+			}
+			if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+				t.Errorf("expected port to be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
+			}
+
+			_, _, err = storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}})
+			if err != nil {
+				t.Fatalf("unexpected error deleting service: %v", err)
+			}
+
+			// Ensure they are still allocated.
+			for i, fam := range createdSvc.Spec.IPFamilies {
+				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+					t.Errorf("expected IP to still be allocated: %q", createdSvc.Spec.ClusterIPs[i])
+				}
+			}
+			for _, p := range createdSvc.Spec.Ports {
+				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+					t.Errorf("expected port to still be allocated: %d", p.NodePort)
+				}
+			}
+			if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+				t.Errorf("expected port to still be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
+			}
+		})
+	}
+}

From cb4d8700d34500ef7ff336fe9b74fa4a1d90d401 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 11 Jul 2021 12:09:29 -0700
Subject: [PATCH 26/79] Svc REST: Clean up redundant delete tests

---
 .../core/service/storage/rest_test.go         | 110 ------------------
 .../core/service/storage/storage_test.go      |  58 +++++++++
 2 files changed, 58 insertions(+), 110 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 0e805bc3df2..0b70abeb2df 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -700,116 +700,6 @@ func TestServiceStorageValidatesUpdate(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryDelete(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("foo")
-	_, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	_, _, err = storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-}
-
-func TestServiceRegistryDeleteDryRun(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	// Test dry run delete request with cluster ip
-	svc := svctest.MakeService("foo")
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	createdSvc := obj.(*api.Service)
-	if createdSvc.Spec.ClusterIP == "" {
-		t.Fatalf("expected ClusterIP to be set")
-	}
-	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], createdSvc.Spec.ClusterIP) {
-		t.Errorf("expected ClusterIP to be allocated")
-	}
-	_, _, err = storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], createdSvc.Spec.ClusterIP) {
-		t.Errorf("unexpected side effect: ip unallocated")
-	}
-
-	// Test dry run delete request with node port
-	svc = svctest.MakeService("foo2", svctest.SetTypeNodePort)
-	obj, err = storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	createdSvc = obj.(*api.Service)
-	if createdSvc.Spec.Ports[0].NodePort == 0 {
-		t.Fatalf("expected NodePort to be set")
-	}
-	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
-		t.Errorf("expected NodePort to be allocated")
-	}
-
-	isValidClusterIPFields(t, storage, svc, createdSvc)
-
-	_, _, err = storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.Ports[0].NodePort) {
-		t.Errorf("unexpected side effect: NodePort unallocated")
-	}
-}
-
-func TestDualStackServiceRegistryDeleteDryRun(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-
-	// dry run for non dualstack
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-	dualstack_storage, dualstack_server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
-	defer dualstack_server.Terminate(t)
-	// Test dry run delete request with cluster ip
-	dualstack_svc := svctest.MakeService("foo",
-		svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
-		svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
-		svctest.SetClusterIPs("2000:0:0:0:0:0:0:1", "1.2.3.4"))
-
-	_, err := dualstack_storage.Create(ctx, dualstack_svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	isValidClusterIPFields(t, dualstack_storage, dualstack_svc, dualstack_svc)
-	_, _, err = dualstack_storage.Delete(ctx, dualstack_svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	for i, family := range dualstack_svc.Spec.IPFamilies {
-		if !ipIsAllocated(t, dualstack_storage.alloc.serviceIPAllocatorsByFamily[family], dualstack_svc.Spec.ClusterIPs[i]) {
-			t.Errorf("unexpected side effect: ip unallocated %v", dualstack_svc.Spec.ClusterIPs[i])
-		}
-	}
-}
-
-func TestServiceRegistryDeleteExternalName(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("foo", svctest.SetTypeExternalName)
-	_, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	_, _, err = storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-}
-
 func TestServiceRegistryUpdateLoadBalancerService(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 88167e379b0..c960e5c4ca2 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5595,6 +5595,64 @@ func TestCreateDryRun(t *testing.T) {
 	}
 }
 
+func TestDeleteTypes(t *testing.T) {
+	testCases := []struct {
+		name string
+		svc  *api.Service
+	}{{
+		name: "type:ExternalName",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeExternalName),
+	}, {
+		name: "type:ClusterIP",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP),
+	}, {
+		name: "type:ClusterIP_headless",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetHeadless),
+	}, {
+		name: "type:NodePort",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeNodePort),
+	}, {
+		name: "type:LoadBalancer",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer),
+	}, {
+		name: "type:LoadBalancer_etp:Local",
+		svc: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+	}}
+
+	// This test is ONLY with the gate enabled.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := genericapirequest.NewDefaultContext()
+			_, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+
+			_, deleted, err := storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error deleting service: %v", err)
+			}
+			if !deleted {
+				t.Fatalf("expected service to be deleted")
+			}
+		})
+	}
+}
+
 // Prove that a dry-run delete doesn't actually deallocate IPs or ports.
 func TestDeleteDryRun(t *testing.T) {
 	testCases := []struct {

From 89a9ca52bcc766061472c2aa436b454b2fdd18c5 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Thu, 15 Jul 2021 22:39:50 -0700
Subject: [PATCH 27/79] Svc REST: Add a delete-with-finalizer test

This is a long-standing bug that gets fixed "for free" in the
de-layering.
---
 .../core/service/storage/storage_test.go      | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index c960e5c4ca2..2acb9dceace 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -23,6 +23,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
@@ -5653,6 +5654,112 @@ func TestDeleteTypes(t *testing.T) {
 	}
 }
 
+func TestDeleteWithFinalizer(t *testing.T) {
+	svcName := "foo"
+
+	// This test is ONLY with the gate enabled.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	// This will allocate cluster IPs, NodePort, and HealthCheckNodePort.
+	svc := svctest.MakeService(svcName, svctest.SetTypeLoadBalancer,
+		svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+		func(s *api.Service) {
+			s.Finalizers = []string{"example.com/test"}
+		})
+
+	ctx := genericapirequest.NewDefaultContext()
+
+	// Create it with finalizer.
+	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error creating service: %v", err)
+	}
+	createdSvc := obj.(*api.Service)
+
+	// Prove everything was allocated.
+	obj, err = storage.Get(ctx, svcName, &metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error getting service: %v", err)
+	}
+	if !cmp.Equal(createdSvc, obj) {
+		t.Errorf("expected the result of Create() and Get() to match: %v", cmp.Diff(createdSvc, obj))
+	}
+	for i, fam := range createdSvc.Spec.IPFamilies {
+		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+			t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
+		}
+	}
+	for _, p := range createdSvc.Spec.Ports {
+		if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+			t.Errorf("expected port to be allocated: %v", p.NodePort)
+		}
+	}
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+		t.Errorf("expected port to be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
+	}
+
+	// Try to delete it, but it should be blocked by the finalizer.
+	obj, deleted, err := storage.Delete(ctx, svcName, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error deleting service: %v", err)
+	}
+	if deleted {
+		t.Fatalf("expected service to not be deleted")
+	}
+	deletedSvc := obj.(*api.Service)
+
+	// Prove everything is still allocated.
+	_, err = storage.Get(ctx, svcName, &metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error getting service: %v", err)
+	}
+	for i, fam := range createdSvc.Spec.IPFamilies {
+		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+			t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
+		}
+	}
+	for _, p := range createdSvc.Spec.Ports {
+		if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+			t.Errorf("expected port to be allocated: %v", p.NodePort)
+		}
+	}
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+		t.Errorf("expected port to be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
+	}
+
+	// Clear the finalizer - should delete.
+	deletedSvc.Finalizers = nil
+	_, _, err = storage.Update(ctx, svcName,
+		rest.DefaultUpdatedObjectInfo(deletedSvc), rest.ValidateAllObjectFunc,
+		rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error updating service: %v", err)
+	}
+
+	// Prove everything is deallocated.
+	_, err = storage.Get(ctx, svcName, &metav1.GetOptions{})
+	if err == nil {
+		t.Fatalf("unexpected success getting service")
+	}
+	for i, fam := range createdSvc.Spec.IPFamilies {
+		if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+			t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIPs[i])
+		}
+	}
+	for _, p := range createdSvc.Spec.Ports {
+		if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+			t.Errorf("expected port to not be allocated: %v", p.NodePort)
+		}
+	}
+	if portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+		t.Errorf("expected port to not be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
+	}
+}
+
 // Prove that a dry-run delete doesn't actually deallocate IPs or ports.
 func TestDeleteDryRun(t *testing.T) {
 	testCases := []struct {

From ccf337657075cd93619c25071f38840ce1acd3b8 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 5 Dec 2020 15:56:37 -0800
Subject: [PATCH 28/79] Svc REST: De-layer Update

This is the last layered method.  All allocator logic is moved to the
beginUpdate() path.  Removing the now-useless layer will happen in a
subsequent commit.
---
 pkg/apis/core/validation/validation.go        |  15 ++
 pkg/registry/core/service/storage/rest.go     | 189 ++++++++----------
 .../core/service/storage/rest_test.go         |  20 +-
 pkg/registry/core/service/storage/storage.go  |  12 ++
 .../core/service/storage/storage_test.go      |   2 +
 pkg/registry/core/service/strategy.go         |  13 +-
 pkg/registry/core/service/strategy_test.go    |  70 -------
 7 files changed, 138 insertions(+), 183 deletions(-)

diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go
index d5c66ce84c4..696563223b8 100644
--- a/pkg/apis/core/validation/validation.go
+++ b/pkg/apis/core/validation/validation.go
@@ -45,6 +45,7 @@ import (
 	schedulinghelper "k8s.io/component-helpers/scheduling/corev1"
 	apiservice "k8s.io/kubernetes/pkg/api/service"
 	"k8s.io/kubernetes/pkg/apis/core"
+	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/core/helper"
 	podshelper "k8s.io/kubernetes/pkg/apis/core/pods"
 	corev1 "k8s.io/kubernetes/pkg/apis/core/v1"
@@ -4528,6 +4529,18 @@ func validateServiceExternalTrafficFieldsValue(service *core.Service) field.Erro
 	return allErrs
 }
 
+func validateServiceExternalTrafficFieldsUpdate(before, after *api.Service) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if apiservice.NeedsHealthCheck(before) && apiservice.NeedsHealthCheck(after) {
+		if after.Spec.HealthCheckNodePort != before.Spec.HealthCheckNodePort {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "healthCheckNodePort"), "field is immutable"))
+		}
+	}
+
+	return allErrs
+}
+
 // validateServiceInternalTrafficFieldsValue validates InternalTraffic related
 // spec have legal value.
 func validateServiceInternalTrafficFieldsValue(service *core.Service) field.ErrorList {
@@ -4592,6 +4605,8 @@ func ValidateServiceUpdate(service, oldService *core.Service) field.ErrorList {
 	upgradeDowngradeLoadBalancerClassErrs := validateLoadBalancerClassField(oldService, service)
 	allErrs = append(allErrs, upgradeDowngradeLoadBalancerClassErrs...)
 
+	allErrs = append(allErrs, validateServiceExternalTrafficFieldsUpdate(oldService, service)...)
+
 	return append(allErrs, ValidateService(service)...)
 }
 
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 06304661746..1e2730ccc4a 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -35,7 +35,6 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
-	"k8s.io/apiserver/pkg/util/dryrun"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 	apiservice "k8s.io/kubernetes/pkg/api/service"
@@ -253,7 +252,6 @@ func (al *RESTAllocStuff) healthCheckNodePortUpdate(oldService, service *api.Ser
 	oldHealthCheckNodePort := oldService.Spec.HealthCheckNodePort
 
 	needsHealthCheckNodePort := apiservice.NeedsHealthCheck(service)
-	newHealthCheckNodePort := service.Spec.HealthCheckNodePort
 
 	switch {
 	// Case 1: Transition from don't need HealthCheckNodePort to needs HealthCheckNodePort.
@@ -271,130 +269,88 @@ func (al *RESTAllocStuff) healthCheckNodePortUpdate(oldService, service *api.Ser
 		klog.Infof("Transition to non LoadBalancer type service or LoadBalancer type service with ExternalTrafficPolicy=Global")
 		klog.V(4).Infof("Releasing healthCheckNodePort: %d", oldHealthCheckNodePort)
 		nodePortOp.ReleaseDeferred(int(oldHealthCheckNodePort))
-
-	// Case 3: Remain in needs HealthCheckNodePort.
-	// Reject changing the value of the HealthCheckNodePort field.
-	case neededHealthCheckNodePort && needsHealthCheckNodePort:
-		if oldHealthCheckNodePort != newHealthCheckNodePort {
-			//FIXME: Let validation do this.
-			klog.Warningf("Attempt to change value of health check node port DENIED")
-			fldPath := field.NewPath("spec", "healthCheckNodePort")
-			el := field.ErrorList{field.Invalid(fldPath, newHealthCheckNodePort,
-				"cannot change healthCheckNodePort on loadBalancer service with externalTraffic=Local during update")}
-			return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
-		}
 	}
 	return true, nil
 }
 
 func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
-	oldObj, err := rs.services.Get(ctx, name, &metav1.GetOptions{})
-	if err != nil {
-		// Support create on update, if forced to.
-		if forceAllowCreate {
-			obj, err := objInfo.UpdatedObject(ctx, nil)
-			if err != nil {
-				return nil, false, err
-			}
-			createdObj, err := rs.Create(ctx, obj, createValidation, &metav1.CreateOptions{DryRun: options.DryRun})
-			if err != nil {
-				return nil, false, err
-			}
-			return createdObj, true, nil
-		}
-		return nil, false, err
-	}
-	oldService := oldObj.(*api.Service)
-	obj, err := objInfo.UpdatedObject(ctx, oldService)
-	if err != nil {
-		return nil, false, err
+	return rs.services.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+}
+
+func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRun bool) (transaction, error) {
+	result := metaTransaction{}
+
+	// Ensure IP family fields are correctly initialized.  We do it here, since
+	// we want this to be visible even when dryRun == true.
+	if err := al.initIPFamilyFields(oldService, service); err != nil {
+		return nil, err
 	}
 
-	service := obj.(*api.Service)
-
-	if !rest.ValidNamespace(ctx, &service.ObjectMeta) {
-		return nil, false, errors.NewConflict(api.Resource("services"), service.Namespace, fmt.Errorf("Service.Namespace does not match the provided context"))
-	}
-
-	// Copy over non-user fields
-	if err := rest.BeforeUpdate(rs.strategy, ctx, service, oldService); err != nil {
-		return nil, false, err
-	}
-
-	var allocated map[api.IPFamily]string
-	var toReleaseIPs map[api.IPFamily]string
-
-	performRelease := false // when set, any clusterIP that should be released will be released
-	// cleanup
-	// on failure: Any allocated ip must be released back
-	// on failure: any ip that should be released, will *not* be released
-	// on success: any ip that should be released, will  be released
-	defer func() {
-		//FIXME: plumb dryRun down here
-		// release the allocated, this is expected to be cleared if the entire function ran to success
-		if allocated_released, err := rs.alloc.releaseClusterIPs(allocated); err != nil {
-			klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v", service.Namespace, service.Name, err, allocated, allocated_released)
-
-		}
-		// performRelease is set when the enture function ran to success
-		if performRelease {
-			//FIXME: plumb dryRun down here
-			if toReleaseIPs_released, err := rs.alloc.releaseClusterIPs(toReleaseIPs); err != nil {
-				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v", service.Namespace, service.Name, err, toReleaseIPs, toReleaseIPs_released)
-			}
-		}
-	}()
-
-	nodePortOp := portallocator.StartOperation(rs.alloc.serviceNodePorts, dryrun.IsDryRun(options.DryRun))
-	defer nodePortOp.Finish()
-
-	// try set ip families (for missing ip families)
-	if err := rs.alloc.initIPFamilyFields(oldService, service); err != nil {
-		return nil, false, err
-	}
-
-	if !dryrun.IsDryRun(options.DryRun) {
-		allocated, toReleaseIPs, err = rs.alloc.handleClusterIPsForUpdatedService(oldService, service)
-		if err != nil {
-			return nil, false, err
+	// Allocate ClusterIPs
+	//FIXME: we need to put values in, even if dry run - else validation should
+	//not pass.  It does but that should be fixed.  Plumb dryRun thru update
+	//logic.
+	// xref: https://groups.google.com/g/kubernetes-sig-api-machinery/c/_-5TKHXHcXE/m/RfKj7CtzAQAJ
+	if !dryRun {
+		if txn, err := al.allocUpdateServiceClusterIPsNew(service, oldService); err != nil {
+			result.Revert()
+			return nil, err
+		} else {
+			result = append(result, txn)
 		}
 	}
+
+	// Allocate ports
+	if txn, err := al.allocUpdateServiceNodePortsNew(service, oldService, dryRun); err != nil {
+		result.Revert()
+		return nil, err
+	} else {
+		result = append(result, txn)
+	}
+
+	return result, nil
+}
+
+//FIXME: rename and merge with updateNodePorts?
+func (al *RESTAllocStuff) allocUpdateServiceNodePortsNew(service, oldService *api.Service, dryRun bool) (transaction, error) {
+	// The allocator tracks dry-run-ness internally.
+	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
+
+	txn := callbackTransaction{
+		commit: func() {
+			nodePortOp.Commit()
+			// We don't NEED to call Finish() here, but for that package says
+			// to, so for future-safety, we will.
+			nodePortOp.Finish()
+		},
+		revert: func() {
+			// Weirdly named but this will revert if commit wasn't called
+			nodePortOp.Finish()
+		},
+	}
+
 	// Update service from NodePort or LoadBalancer to ExternalName or ClusterIP, should release NodePort if exists.
 	if (oldService.Spec.Type == api.ServiceTypeNodePort || oldService.Spec.Type == api.ServiceTypeLoadBalancer) &&
 		(service.Spec.Type == api.ServiceTypeExternalName || service.Spec.Type == api.ServiceTypeClusterIP) {
 		releaseNodePorts(oldService, nodePortOp)
 	}
+
 	// Update service from any type to NodePort or LoadBalancer, should update NodePort.
 	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
 		if err := updateNodePorts(oldService, service, nodePortOp); err != nil {
-			return nil, false, err
+			txn.Revert()
+			return nil, err
 		}
 	}
-	// Update service from LoadBalancer to non-LoadBalancer, should remove any LoadBalancerStatus.
-	if service.Spec.Type != api.ServiceTypeLoadBalancer {
-		// Although loadbalancer delete is actually asynchronous, we don't need to expose the user to that complexity.
-		service.Status.LoadBalancer = api.LoadBalancerStatus{}
-	}
 
 	// Handle ExternalTraffic related updates.
-	success, err := rs.alloc.healthCheckNodePortUpdate(oldService, service, nodePortOp)
+	success, err := al.healthCheckNodePortUpdate(oldService, service, nodePortOp)
 	if !success || err != nil {
-		return nil, false, err
+		txn.Revert()
+		return nil, err
 	}
 
-	out, created, err := rs.services.Update(ctx, service.Name, rest.DefaultUpdatedObjectInfo(service), createValidation, updateValidation, forceAllowCreate, options)
-	if err == nil {
-		el := nodePortOp.Commit()
-		if el != nil {
-			// problems should be fixed by an eventual reconciliation / restart
-			utilruntime.HandleError(fmt.Errorf("error(s) committing NodePorts changes: %v", el))
-		}
-	}
-	// all good
-	allocated = nil       // if something was allocated, keep it allocated
-	performRelease = true // if something that should be released then go ahead and release it
-
-	return out, created, err
+	return txn, nil
 }
 
 // GetResetFields implements rest.ResetFieldsStrategy
@@ -635,6 +591,35 @@ func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service, dryRun bo
 	return allocated, err
 }
 
+//FIXME: rename and merge with handleClusterIPsForUpdatedService
+func (al *RESTAllocStuff) allocUpdateServiceClusterIPsNew(service *api.Service, oldService *api.Service) (transaction, error) {
+	allocated, released, err := al.handleClusterIPsForUpdatedService(oldService, service)
+	if err != nil {
+		return nil, err
+	}
+
+	// on failure: Any newly allocated IP must be released back
+	// on failure: Any previously allocated IP that would have been released,
+	//             must *not* be released
+	// on success: Any previously allocated IP that should be released, will be
+	//             released
+	txn := callbackTransaction{
+		commit: func() {
+			if actuallyReleased, err := al.releaseClusterIPs(released); err != nil {
+				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v",
+					service.Namespace, service.Name, err, released, actuallyReleased)
+			}
+		},
+		revert: func() {
+			if actuallyReleased, err := al.releaseClusterIPs(allocated); err != nil {
+				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v",
+					service.Namespace, service.Name, err, allocated, actuallyReleased)
+			}
+		},
+	}
+	return txn, nil
+}
+
 // handles type change/upgrade/downgrade change type for an update service
 // this func does not perform actual release of clusterIPs. it returns
 // a map[family]ip for the caller to release when everything else has
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 0b70abeb2df..838e08483f7 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -133,14 +133,17 @@ func (s *serviceStorage) Create(ctx context.Context, obj runtime.Object, createV
 }
 
 func (s *serviceStorage) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
-	obj, err := objInfo.UpdatedObject(ctx, nil)
+	ret, created, err := s.inner.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
 	if err != nil {
-		return nil, false, err
+		return ret, created, err
 	}
-	if !dryrun.IsDryRun(options.DryRun) {
-		s.saveService(obj.(*api.Service))
+	if dryrun.IsDryRun(options.DryRun) {
+		return ret.DeepCopyObject(), created, err
 	}
-	return obj, false, nil
+	svc := ret.(*api.Service)
+	s.saveService(svc)
+
+	return s.Services[name].DeepCopy(), created, nil
 }
 
 func (s *serviceStorage) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
@@ -716,12 +719,13 @@ func TestServiceRegistryUpdateLoadBalancerService(t *testing.T) {
 	svc2 := obj.(*api.Service).DeepCopy()
 	svc2.Spec.Type = api.ServiceTypeLoadBalancer
 	svc2.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
-	if _, _, err := storage.Update(ctx, svc2.Name, rest.DefaultUpdatedObjectInfo(svc2), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil {
+	obj, _, err = storage.Update(ctx, svc2.Name, rest.DefaultUpdatedObjectInfo(svc2), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
+	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
 	}
 
 	// Change port.
-	svc3 := svc2.DeepCopy()
+	svc3 := obj.(*api.Service).DeepCopy()
 	svc3.Spec.Ports[0].Port = 6504
 	if _, _, err := storage.Update(ctx, svc3.Name, rest.DefaultUpdatedObjectInfo(svc3), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -971,7 +975,7 @@ func TestServiceRegistryIPUpdate(t *testing.T) {
 		}
 	}
 
-	update = createdService.DeepCopy()
+	update = updatedService.DeepCopy()
 	update.Spec.Ports[0].Port = 6503
 	update.Spec.ClusterIP = testIP
 	update.Spec.ClusterIPs[0] = testIP
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 6e18ad27d97..a385fc7a96e 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -326,15 +326,27 @@ func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Objec
 	newSvc := obj.(*api.Service)
 	oldSvc := oldObj.(*api.Service)
 
+	// Fix up allocated values that the client may have not specified (for
+	// idempotence).
+	svcreg.PatchAllocatedValues(newSvc, oldSvc)
+
 	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
 	// early, before anyone looks at them.
 	// NOTE: the args are (old, new)
 	svcreg.NormalizeClusterIPs(oldSvc, newSvc)
 
+	// Allocate and initialize fields.
+	txn, err := r.alloc.allocateUpdate(newSvc, oldSvc, dryrun.IsDryRun(options.DryRun))
+	if err != nil {
+		return nil, err
+	}
+
 	// Our cleanup callback
 	finish := func(_ context.Context, success bool) {
 		if success {
+			txn.Commit()
 		} else {
+			txn.Revert()
 		}
 	}
 
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 2acb9dceace..ab31df2e5a1 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5837,3 +5837,5 @@ func TestDeleteDryRun(t *testing.T) {
 		})
 	}
 }
+
+//FIXME: Tests for update()
diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go
index 3439b423924..0a34b59876c 100644
--- a/pkg/registry/core/service/strategy.go
+++ b/pkg/registry/core/service/strategy.go
@@ -120,7 +120,6 @@ func (strategy svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runti
 	oldService := old.(*api.Service)
 	newService.Status = oldService.Status
 
-	patchAllocatedValues(newService, oldService)
 	//FIXME: Normalize is now called from BeginUpdate in pkg/registry/core/service/storage
 	NormalizeClusterIPs(oldService, newService)
 	dropServiceDisabledFields(newService, oldService)
@@ -305,11 +304,12 @@ func (serviceStatusStrategy) WarningsOnUpdate(ctx context.Context, obj, old runt
 	return nil
 }
 
-// patchAllocatedValues allows clients to avoid a read-modify-write cycle while
+// PatchAllocatedValues allows clients to avoid a read-modify-write cycle while
 // preserving values that we allocated on their behalf.  For example, they
 // might create a Service without specifying the ClusterIP, in which case we
 // allocate one.  If they resubmit that same YAML, we want it to succeed.
-func patchAllocatedValues(newSvc, oldSvc *api.Service) {
+//FIXME: move this to pkg/registry/core/service/storage
+func PatchAllocatedValues(newSvc, oldSvc *api.Service) {
 	if needsClusterIP(oldSvc) && needsClusterIP(newSvc) {
 		if newSvc.Spec.ClusterIP == "" {
 			newSvc.Spec.ClusterIP = oldSvc.Spec.ClusterIP
@@ -521,6 +521,13 @@ func dropTypeDependentFields(newSvc *api.Service, oldSvc *api.Service) {
 	// NOTE: there are other fields like `selector` which we could wipe.
 	// Historically we did not wipe them and they are not allocated from
 	// finite pools, so we are (currently) choosing to leave them alone.
+
+	// Clear the load-balancer status if it is no longer appropriate.  Although
+	// LB de-provisioning is actually asynchronous, we don't need to expose the
+	// user to that complexity.
+	if newSvc.Spec.Type != api.ServiceTypeLoadBalancer {
+		newSvc.Status.LoadBalancer = api.LoadBalancerStatus{}
+	}
 }
 
 func needsClusterIP(svc *api.Service) bool {
diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go
index bc5759788d9..c405c334941 100644
--- a/pkg/registry/core/service/strategy_test.go
+++ b/pkg/registry/core/service/strategy_test.go
@@ -22,7 +22,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -124,75 +123,6 @@ func makeServiceWithClusterIp(clusterIP string, clusterIPs []string) *api.Servic
 	}
 }
 
-// TODO: This should be done on types that are not part of our API
-func TestBeforeUpdate(t *testing.T) {
-	testCases := []struct {
-		name      string
-		tweakSvc  func(oldSvc, newSvc *api.Service) // given basic valid services, each test case can customize them
-		expectErr bool
-	}{
-		{
-			name: "no change",
-			tweakSvc: func(oldSvc, newSvc *api.Service) {
-				// nothing
-			},
-			expectErr: false,
-		},
-		{
-			name: "change port",
-			tweakSvc: func(oldSvc, newSvc *api.Service) {
-				newSvc.Spec.Ports[0].Port++
-			},
-			expectErr: false,
-		},
-		{
-			name: "bad namespace",
-			tweakSvc: func(oldSvc, newSvc *api.Service) {
-				newSvc.Namespace = "#$%%invalid"
-			},
-			expectErr: true,
-		},
-		{
-			name: "change name",
-			tweakSvc: func(oldSvc, newSvc *api.Service) {
-				newSvc.Name += "2"
-			},
-			expectErr: true,
-		},
-		{
-			name: "change ClusterIP",
-			tweakSvc: func(oldSvc, newSvc *api.Service) {
-				oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"}
-				newSvc.Spec.ClusterIPs = []string{"4.3.2.1"}
-			},
-			expectErr: true,
-		},
-		{
-			name: "change selector",
-			tweakSvc: func(oldSvc, newSvc *api.Service) {
-				newSvc.Spec.Selector = map[string]string{"newkey": "newvalue"}
-			},
-			expectErr: false,
-		},
-	}
-
-	for _, tc := range testCases {
-		strategy, _ := newStrategy("172.30.0.0/16", false)
-
-		oldSvc := makeValidService()
-		newSvc := makeValidService()
-		tc.tweakSvc(oldSvc, newSvc)
-		ctx := genericapirequest.NewDefaultContext()
-		err := rest.BeforeUpdate(strategy, ctx, runtime.Object(oldSvc), runtime.Object(newSvc))
-		if tc.expectErr && err == nil {
-			t.Errorf("unexpected non-error for %q", tc.name)
-		}
-		if !tc.expectErr && err != nil {
-			t.Errorf("unexpected error for %q: %v", tc.name, err)
-		}
-	}
-}
-
 func TestServiceStatusStrategy(t *testing.T) {
 	_, testStatusStrategy := newStrategy("10.0.0.0/16", false)
 	ctx := genericapirequest.NewDefaultContext()

From 30bd8198e30265ef7a74b2608b68f897010971a2 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 6 Aug 2021 22:43:30 -0700
Subject: [PATCH 29/79] Svc REST: Set Cluster IPs during dry-run Update()

Dry-run should return valid results.

Also add a test.
---
 pkg/registry/core/service/storage/rest.go     |  36 ++--
 .../core/service/storage/rest_test.go         |  86 --------
 .../core/service/storage/storage_test.go      | 195 ++++++++++++++++++
 3 files changed, 214 insertions(+), 103 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 1e2730ccc4a..8948207722c 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -287,17 +287,14 @@ func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRu
 	}
 
 	// Allocate ClusterIPs
-	//FIXME: we need to put values in, even if dry run - else validation should
-	//not pass.  It does but that should be fixed.  Plumb dryRun thru update
-	//logic.
-	// xref: https://groups.google.com/g/kubernetes-sig-api-machinery/c/_-5TKHXHcXE/m/RfKj7CtzAQAJ
-	if !dryRun {
-		if txn, err := al.allocUpdateServiceClusterIPsNew(service, oldService); err != nil {
-			result.Revert()
-			return nil, err
-		} else {
-			result = append(result, txn)
-		}
+	//TODO(thockin): validation should not pass with empty clusterIP, but it
+	//does (and is tested!).  Fixing that all is a big PR and will have to
+	//happen later.
+	if txn, err := al.allocUpdateServiceClusterIPsNew(service, oldService, dryRun); err != nil {
+		result.Revert()
+		return nil, err
+	} else {
+		result = append(result, txn)
 	}
 
 	// Allocate ports
@@ -592,8 +589,8 @@ func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service, dryRun bo
 }
 
 //FIXME: rename and merge with handleClusterIPsForUpdatedService
-func (al *RESTAllocStuff) allocUpdateServiceClusterIPsNew(service *api.Service, oldService *api.Service) (transaction, error) {
-	allocated, released, err := al.handleClusterIPsForUpdatedService(oldService, service)
+func (al *RESTAllocStuff) allocUpdateServiceClusterIPsNew(service *api.Service, oldService *api.Service, dryRun bool) (transaction, error) {
+	allocated, released, err := al.handleClusterIPsForUpdatedService(oldService, service, dryRun)
 	if err != nil {
 		return nil, err
 	}
@@ -605,12 +602,18 @@ func (al *RESTAllocStuff) allocUpdateServiceClusterIPsNew(service *api.Service,
 	//             released
 	txn := callbackTransaction{
 		commit: func() {
+			if dryRun {
+				return
+			}
 			if actuallyReleased, err := al.releaseClusterIPs(released); err != nil {
 				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v",
 					service.Namespace, service.Name, err, released, actuallyReleased)
 			}
 		},
 		revert: func() {
+			if dryRun {
+				return
+			}
 			if actuallyReleased, err := al.releaseClusterIPs(allocated); err != nil {
 				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v",
 					service.Namespace, service.Name, err, allocated, actuallyReleased)
@@ -624,7 +627,7 @@ func (al *RESTAllocStuff) allocUpdateServiceClusterIPsNew(service *api.Service,
 // this func does not perform actual release of clusterIPs. it returns
 // a map[family]ip for the caller to release when everything else has
 // executed successfully
-func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Service, service *api.Service) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Service, service *api.Service, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
 
 	// We don't want to upgrade (add an IP) or downgrade (remove an IP)
 	// following a cluster downgrade/upgrade to/from dual-stackness
@@ -647,8 +650,7 @@ func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Serv
 	// CASE A:
 	// Update service from ExternalName to non-ExternalName, should initialize ClusterIP.
 	if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName {
-		//FIXME: plumb dryRun down here
-		allocated, err := al.allocServiceClusterIPs(service, false)
+		allocated, err := al.allocServiceClusterIPs(service, dryRun)
 		return allocated, nil, err
 	}
 
@@ -694,7 +696,7 @@ func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Serv
 		toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1]
 
 		// allocate
-		allocated, err := al.allocClusterIPs(service, toAllocate, false) //FIXME: plumb dry-run down here
+		allocated, err := al.allocClusterIPs(service, toAllocate, dryRun)
 		// set if successful
 		if err == nil {
 			service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]]
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 838e08483f7..4a6b9d015e5 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -592,92 +592,6 @@ func TestServiceRegistryUpdateUnspecifiedAllocations(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryUpdateDryRun(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	obj, err := storage.Create(ctx, svctest.MakeService("foo", svctest.SetTypeExternalName), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	svc := obj.(*api.Service)
-
-	// Test dry run update request external name to node port
-	new1 := svc.DeepCopy()
-	svctest.SetTypeNodePort(new1)
-	svctest.SetNodePorts(30001)(new1) // DryRun does not set port values yet
-	obj, created, err := storage.Update(ctx, new1.Name, rest.DefaultUpdatedObjectInfo(new1),
-		rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	if obj == nil {
-		t.Errorf("Expected non-nil object")
-	}
-	if created {
-		t.Errorf("expected not created")
-	}
-	if portIsAllocated(t, storage.alloc.serviceNodePorts, new1.Spec.Ports[0].NodePort) {
-		t.Errorf("unexpected side effect: NodePort allocated")
-	}
-
-	// Test dry run update request external name to cluster ip
-	new2 := svc.DeepCopy()
-	svctest.SetTypeClusterIP(new2)
-	svctest.SetClusterIPs("1.2.3.4")(new2) // DryRun does not set IP values yet
-	_, _, err = storage.Update(ctx, svc.Name, rest.DefaultUpdatedObjectInfo(new2),
-		rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], new2.Spec.ClusterIP) {
-		t.Errorf("unexpected side effect: ip allocated")
-	}
-
-	// Test dry run update request remove node port
-	obj, err = storage.Create(ctx, svctest.MakeService("foo2", svctest.SetTypeNodePort, svctest.SetNodePorts(30001)), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	svc = obj.(*api.Service)
-	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], svc.Spec.ClusterIP) {
-		t.Errorf("expected IP to be allocated")
-	}
-	if !portIsAllocated(t, storage.alloc.serviceNodePorts, svc.Spec.Ports[0].NodePort) {
-		t.Errorf("expected NodePort to be allocated")
-	}
-
-	new3 := svc.DeepCopy()
-	svctest.SetTypeExternalName(new3)
-	_, _, err = storage.Update(ctx, svc.Name, rest.DefaultUpdatedObjectInfo(new3),
-		rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	if !portIsAllocated(t, storage.alloc.serviceNodePorts, svc.Spec.Ports[0].NodePort) {
-		t.Errorf("unexpected side effect: NodePort unallocated")
-	}
-
-	// Test dry run update request remove cluster ip
-	obj, err = storage.Create(ctx, svctest.MakeService("foo3", svctest.SetClusterIPs("1.2.3.4")), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("expected no error: %v", err)
-	}
-	svc = obj.(*api.Service)
-
-	new4 := svc.DeepCopy()
-	svctest.SetTypeExternalName(new4)
-	_, _, err = storage.Update(ctx, svc.Name, rest.DefaultUpdatedObjectInfo(new4),
-		rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{DryRun: []string{metav1.DryRunAll}})
-	if err != nil {
-		t.Fatalf("expected no error: %v", err)
-	}
-	if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily], svc.Spec.ClusterIP) {
-		t.Errorf("unexpected side effect: ip unallocated")
-	}
-}
-
 func TestServiceStorageValidatesUpdate(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index ab31df2e5a1..da3aab0d269 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5838,4 +5838,199 @@ func TestDeleteDryRun(t *testing.T) {
 	}
 }
 
+// Prove that a dry-run update doesn't actually allocate or deallocate IPs or ports.
+func TestUpdateDryRun(t *testing.T) {
+	testCases := []struct {
+		name            string
+		clusterFamilies []api.IPFamily
+		svc             *api.Service
+		update          *api.Service
+		verifyDryAllocs bool
+	}{{
+		name:            "singlestack:v4_NoAllocs-Allocs",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		svc:             svctest.MakeService("foo", svctest.SetTypeExternalName),
+		update: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		verifyDryAllocs: true, // make sure values were not allocated.
+	}, {
+		name:            "singlestack:v4_Allocs-NoAllocs",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		update:          svctest.MakeService("foo", svctest.SetTypeExternalName),
+		verifyDryAllocs: false, // make sure values were not released.
+	}, {
+		name:            "singlestack:v6_NoAllocs-Allocs",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		svc:             svctest.MakeService("foo", svctest.SetTypeExternalName),
+		update: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		verifyDryAllocs: true, // make sure values were not allocated.
+	}, {
+		name:            "singlestack:v6_Allocs-NoAllocs",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol},
+		svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		update:          svctest.MakeService("foo", svctest.SetTypeExternalName),
+		verifyDryAllocs: false, // make sure values were not released.
+	}, {
+		name:            "dualstack:v4v6_NoAllocs-Allocs",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		svc:             svctest.MakeService("foo", svctest.SetTypeExternalName),
+		update: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		verifyDryAllocs: true, // make sure values were not allocated.
+	}, {
+		name:            "dualstack:v4v6_Allocs-NoAllocs",
+		clusterFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		update:          svctest.MakeService("foo", svctest.SetTypeExternalName),
+		verifyDryAllocs: false, // make sure values were not released.
+	}, {
+		name:            "dualstack:v6v4_NoAllocs-Allocs",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+		svc:             svctest.MakeService("foo", svctest.SetTypeExternalName),
+		update: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		verifyDryAllocs: true, // make sure values were not allocated.
+	}, {
+		name:            "dualstack:v6v4_Allocs-NoAllocs",
+		clusterFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+		svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		update:          svctest.MakeService("foo", svctest.SetTypeExternalName),
+		verifyDryAllocs: false, // make sure values were not released.
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			storage, _, server := newStorage(t, tc.clusterFamilies)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			ctx := genericapirequest.NewDefaultContext()
+			obj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			createdSvc := obj.(*api.Service)
+
+			if tc.verifyDryAllocs {
+				// Dry allocs means no allocs on create.  Ensure values were
+				// NOT allocated.
+				for i, fam := range createdSvc.Spec.IPFamilies {
+					if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+						t.Errorf("expected IP to not be allocated: %q", createdSvc.Spec.ClusterIPs[i])
+					}
+				}
+				for _, p := range createdSvc.Spec.Ports {
+					if p.NodePort != 0 {
+						t.Errorf("expected port to not be assigned: %d", p.NodePort)
+						if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+							t.Errorf("expected port to not be allocated: %d", p.NodePort)
+						}
+					}
+				}
+				if createdSvc.Spec.HealthCheckNodePort != 0 {
+					t.Errorf("expected HCNP to not be assigned: %d", createdSvc.Spec.HealthCheckNodePort)
+					if portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+						t.Errorf("expected HCNP to not be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
+					}
+				}
+			} else {
+				// Ensure IPs were allocated
+				for i, fam := range createdSvc.Spec.IPFamilies {
+					if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+						t.Errorf("expected IP to be allocated: %q", createdSvc.Spec.ClusterIPs[i])
+					}
+				}
+				for _, p := range createdSvc.Spec.Ports {
+					if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+						t.Errorf("expected port to be allocated: %d", p.NodePort)
+					}
+				}
+				if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+					t.Errorf("expected port to be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
+				}
+			}
+
+			// Update the object to the new state and check the results.
+			obj, _, err = storage.Update(ctx, tc.update.Name,
+				rest.DefaultUpdatedObjectInfo(tc.update), rest.ValidateAllObjectFunc,
+				rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{DryRun: []string{metav1.DryRunAll}})
+			if err != nil {
+				t.Fatalf("unexpected error updating service: %v", err)
+			}
+			updatedSvc := obj.(*api.Service)
+
+			if tc.verifyDryAllocs {
+				// Dry allocs means the values are assigned but not
+				// allocated.
+				if netutils.ParseIPSloppy(updatedSvc.Spec.ClusterIP) == nil {
+					t.Errorf("expected valid clusterIP: %q", updatedSvc.Spec.ClusterIP)
+				}
+				for _, ip := range updatedSvc.Spec.ClusterIPs {
+					if netutils.ParseIPSloppy(ip) == nil {
+						t.Errorf("expected valid clusterIP: %q", updatedSvc.Spec.ClusterIP)
+					}
+				}
+				for i, fam := range updatedSvc.Spec.IPFamilies {
+					if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], updatedSvc.Spec.ClusterIPs[i]) {
+						t.Errorf("expected IP to not be allocated: %q", updatedSvc.Spec.ClusterIPs[i])
+					}
+				}
+
+				for _, p := range updatedSvc.Spec.Ports {
+					if p.NodePort == 0 {
+						t.Errorf("expected nodePort to be assigned: %d", p.NodePort)
+					}
+					if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+						t.Errorf("expected nodePort to not be allocated: %d", p.NodePort)
+					}
+				}
+
+				if updatedSvc.Spec.HealthCheckNodePort == 0 {
+					t.Errorf("expected HCNP to be assigned: %d", updatedSvc.Spec.HealthCheckNodePort)
+				}
+				if portIsAllocated(t, storage.alloc.serviceNodePorts, updatedSvc.Spec.HealthCheckNodePort) {
+					t.Errorf("expected HCNP to not be allocated: %d", updatedSvc.Spec.HealthCheckNodePort)
+				}
+			} else {
+				// Ensure IPs were unassigned but not deallocated.
+				if updatedSvc.Spec.ClusterIP != "" {
+					t.Errorf("expected clusterIP to be unset: %q", updatedSvc.Spec.ClusterIP)
+				}
+				if len(updatedSvc.Spec.ClusterIPs) != 0 {
+					t.Errorf("expected clusterIPs to be unset: %q", updatedSvc.Spec.ClusterIPs)
+				}
+				for i, fam := range createdSvc.Spec.IPFamilies {
+					if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
+						t.Errorf("expected IP to still be allocated: %q", createdSvc.Spec.ClusterIPs[i])
+					}
+				}
+
+				for _, p := range updatedSvc.Spec.Ports {
+					if p.NodePort != 0 {
+						t.Errorf("expected nodePort to be unset: %d", p.NodePort)
+					}
+				}
+				for _, p := range createdSvc.Spec.Ports {
+					if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+						t.Errorf("expected nodePort to still be allocated: %d", p.NodePort)
+					}
+				}
+
+				if updatedSvc.Spec.HealthCheckNodePort != 0 {
+					t.Errorf("expected HCNP to be unset: %d", updatedSvc.Spec.HealthCheckNodePort)
+				}
+				if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
+					t.Errorf("expected HCNP to still be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
+				}
+			}
+		})
+	}
+}
+
 //FIXME: Tests for update()

From 446a2c730de48771ea50ad602ee33d23f5fe1873 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 6 Aug 2021 17:26:19 -0700
Subject: [PATCH 30/79] Svc REST: Add a test for PatchAllocatedValues

---
 pkg/registry/core/service/strategy_test.go | 115 ++++++++++++++++++++-
 1 file changed, 112 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go
index c405c334941..5603c90a766 100644
--- a/pkg/registry/core/service/strategy_test.go
+++ b/pkg/registry/core/service/strategy_test.go
@@ -20,17 +20,18 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
-	api "k8s.io/kubernetes/pkg/apis/core"
-	_ "k8s.io/kubernetes/pkg/apis/core/install"
-
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	svctest "k8s.io/kubernetes/pkg/api/service/testing"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	_ "k8s.io/kubernetes/pkg/apis/core/install"
 	"k8s.io/kubernetes/pkg/features"
 	netutils "k8s.io/utils/net"
 	utilpointer "k8s.io/utils/pointer"
@@ -1037,3 +1038,111 @@ func TestTrimFieldsForDualStackDowngrade(t *testing.T) {
 		})
 	}
 }
+
+func TestPatchAllocatedValues(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		before               *api.Service
+		update               *api.Service
+		expectSameClusterIPs bool
+		expectSameNodePort   bool
+		expectSameHCNP       bool
+	}{{
+		name: "all_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		expectSameClusterIPs: true,
+		expectSameNodePort:   true,
+		expectSameHCNP:       true,
+	}, {
+		name: "IPs_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			// these are not valid, but prove the test
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP),
+		expectSameClusterIPs: true,
+	}, {
+		name: "NPs_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			// this is not valid, but proves the test
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetClusterIPs("10.0.0.93", "2000::76")),
+		expectSameClusterIPs: true,
+		expectSameNodePort:   true,
+	}, {
+		name: "HCNP_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts),
+		expectSameClusterIPs: true,
+		expectSameNodePort:   true,
+		expectSameHCNP:       true,
+	}, {
+		name: "nothing_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			// these are not valid, but prove the test
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			update := tc.update.DeepCopy()
+			PatchAllocatedValues(update, tc.before)
+
+			if b, u := tc.before.Spec.ClusterIP, update.Spec.ClusterIP; tc.expectSameClusterIPs && b != u {
+				t.Errorf("expected clusterIP to be patched: %q != %q", b, u)
+			} else if !tc.expectSameClusterIPs && b == u {
+				t.Errorf("expected clusterIP to not be patched: %q == %q", b, u)
+			}
+
+			if b, u := tc.before.Spec.ClusterIPs, update.Spec.ClusterIPs; tc.expectSameClusterIPs && !cmp.Equal(b, u) {
+				t.Errorf("expected clusterIPs to be patched: %q != %q", b, u)
+			} else if !tc.expectSameClusterIPs && cmp.Equal(b, u) {
+				t.Errorf("expected clusterIPs to not be patched: %q == %q", b, u)
+			}
+
+			if b, u := tc.before.Spec.Ports[0].NodePort, update.Spec.Ports[0].NodePort; tc.expectSameNodePort && b != u {
+				t.Errorf("expected nodePort to be patched: %d != %d", b, u)
+			} else if !tc.expectSameNodePort && b == u {
+				t.Errorf("expected nodePort to not be patched: %d == %d", b, u)
+			}
+
+			if b, u := tc.before.Spec.HealthCheckNodePort, update.Spec.HealthCheckNodePort; tc.expectSameHCNP && b != u {
+				t.Errorf("expected healthCheckNodePort to be patched: %d != %d", b, u)
+			} else if !tc.expectSameHCNP && b == u {
+				t.Errorf("expected healthCheckNodePort to not be patched: %d == %d", b, u)
+			}
+		})
+	}
+}

From 5363f1646f9ac819633ae99d5f08a98d3dae34e3 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 11 Jul 2021 22:41:17 -0700
Subject: [PATCH 31/79] Svc REST: Add new model of feature tests

This scaffolding allows us to assert more on each test case, and more
consistently.

Set input fields from output fields IFF they are expected AND not set on
input.  This allows us to verify the "after" state (expected) whether
the test case specified the value or not, and still pass the generic
cmp.Equal.

Use this in a few tests to prove its worth, more to do.

Some of the existing tests that are focused on create and delete can
probably be replaced by these.

This could be used in other test cases that are open-coding a lot of the
same stuff.  Later commits.
---
 pkg/api/service/testing/make.go               |  25 +
 .../core/service/storage/rest_test.go         |  35 -
 .../core/service/storage/storage_test.go      | 965 +++++++++++++++++-
 3 files changed, 989 insertions(+), 36 deletions(-)

diff --git a/pkg/api/service/testing/make.go b/pkg/api/service/testing/make.go
index 6be0abe6c4a..ed0d4ddf347 100644
--- a/pkg/api/service/testing/make.go
+++ b/pkg/api/service/testing/make.go
@@ -205,3 +205,28 @@ func SetHealthCheckNodePort(value int32) Tweak {
 		svc.Spec.HealthCheckNodePort = value
 	}
 }
+
+// SetSessionAffinity sets the SessionAffinity field.
+func SetSessionAffinity(affinity api.ServiceAffinity) Tweak {
+	return func(svc *api.Service) {
+		svc.Spec.SessionAffinity = affinity
+		switch affinity {
+		case api.ServiceAffinityNone:
+			svc.Spec.SessionAffinityConfig = nil
+		case api.ServiceAffinityClientIP:
+			timeout := int32(10)
+			svc.Spec.SessionAffinityConfig = &api.SessionAffinityConfig{
+				ClientIP: &api.ClientIPConfig{
+					TimeoutSeconds: &timeout,
+				},
+			}
+		}
+	}
+}
+
+// SetExternalName sets the ExternalName field.
+func SetExternalName(val string) Tweak {
+	return func(svc *api.Service) {
+		svc.Spec.ExternalName = val
+	}
+}
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 4a6b9d015e5..ae25c59c05c 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -303,41 +303,6 @@ func makeIPNet6(t *testing.T) *net.IPNet {
 	return net
 }
 
-func TestServiceRegistryUpdate(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	_, err := storage.Create(ctx, svctest.MakeService("foo"), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-
-	obj, err := storage.Get(ctx, "foo", &metav1.GetOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error :%v", err)
-	}
-	svc := obj.(*api.Service)
-
-	// update selector
-	svc.Spec.Selector = map[string]string{"bar": "baz2"}
-
-	updatedSvc, created, err := storage.Update(ctx, "foo", rest.DefaultUpdatedObjectInfo(svc), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-	if err != nil {
-		t.Fatalf("Expected no error: %v", err)
-	}
-	if updatedSvc == nil {
-		t.Errorf("Expected non-nil object")
-	}
-	if created {
-		t.Errorf("expected not created")
-	}
-	updatedService := updatedSvc.(*api.Service)
-	if updatedService.Name != "foo" {
-		t.Errorf("Expected foo, but got %v", updatedService.Name)
-	}
-}
-
 func TestServiceRegistryUpdateUnspecifiedAllocations(t *testing.T) {
 	type proof func(t *testing.T, s *api.Service)
 	prove := func(proofs ...proof) []proof {
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index da3aab0d269..2cba09b6fbb 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
@@ -6033,4 +6034,966 @@ func TestUpdateDryRun(t *testing.T) {
 	}
 }
 
-//FIXME: Tests for update()
+type svcTestCase struct {
+	svc         *api.Service
+	expectError bool
+
+	// We could calculate these by looking at the Service, but that's a
+	// vector for test bugs and more importantly it makes the test cases less
+	// self-documenting.
+	expectClusterIPs          bool
+	expectHeadless            bool
+	expectNodePorts           bool
+	expectHealthCheckNodePort bool
+}
+
+func callName(before, after *api.Service) string {
+	if before == nil && after != nil {
+		return "create"
+	}
+	if before != nil && after != nil {
+		return "update"
+	}
+	if before != nil && after == nil {
+		return "delete"
+	}
+	panic("this test is broken: before and after are both nil")
+}
+
+func proveClusterIPsAllocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+	t.Helper()
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
+			t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
+		}
+	}
+
+	clips := []string{}
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		clips = after.Spec.ClusterIPs
+	} else {
+		clips = append(clips, after.Spec.ClusterIP)
+	}
+	for _, clip := range clips {
+		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
+			t.Errorf("%s: expected clusterIP to be allocated: %q", callName(before, after), clip)
+		}
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		if lc, lf := len(after.Spec.ClusterIPs), len(after.Spec.IPFamilies); lc != lf {
+			t.Errorf("%s: expected same number of clusterIPs and ipFamilies: %d != %d", callName(before, after), lc, lf)
+		}
+	}
+
+	for i, fam := range after.Spec.IPFamilies {
+		if want, got := fam, familyOf(after.Spec.ClusterIPs[i]); want != got {
+			t.Errorf("%s: clusterIP is the wrong IP family: want %s, got %s", callName(before, after), want, got)
+		}
+	}
+
+	if before != nil {
+		if before.Spec.ClusterIP != "" {
+			if want, got := before.Spec.ClusterIP, after.Spec.ClusterIP; want != got {
+				t.Errorf("%s: wrong clusterIP: wanted %q, got %q", callName(before, after), want, got)
+			}
+		}
+		for i := range before.Spec.ClusterIPs {
+			if want, got := before.Spec.ClusterIPs[i], after.Spec.ClusterIPs[i]; want != got {
+				t.Errorf("%s: wrong clusterIPs[%d]: wanted %q, got %q", callName(before, after), i, want, got)
+			}
+		}
+		for i := range before.Spec.IPFamilies {
+			if want, got := before.Spec.IPFamilies[i], after.Spec.IPFamilies[i]; want != got {
+				t.Errorf("%s: wrong IPFamilies[%d]: wanted %q, got %q", callName(before, after), i, want, got)
+			}
+		}
+	}
+}
+
+func proveClusterIPsDeallocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+	t.Helper()
+
+	if after != nil && after.Spec.ClusterIP != api.ClusterIPNone {
+		if after.Spec.ClusterIP != "" {
+			t.Errorf("%s: expected clusterIP to be unset: %q", callName(before, after), after.Spec.ClusterIP)
+		}
+		if len(after.Spec.ClusterIPs) != 0 {
+			t.Errorf("%s: expected clusterIPs to be unset: %q", callName(before, after), after.Spec.ClusterIPs)
+		}
+	}
+
+	if before != nil && before.Spec.ClusterIP != api.ClusterIPNone {
+		clips := []string{}
+		if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+			clips = before.Spec.ClusterIPs
+		} else {
+			clips = append(clips, before.Spec.ClusterIP)
+		}
+		for _, clip := range clips {
+			if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
+				t.Errorf("%s: expected clusterIP to be deallocated: %q", callName(before, after), clip)
+			}
+		}
+	}
+}
+
+func proveHeadless(t *testing.T, storage *GenericREST, before, after *api.Service) {
+	t.Helper()
+
+	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
+		t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
+	}
+	if len(after.Spec.ClusterIPs) != 1 || after.Spec.ClusterIPs[0] != api.ClusterIPNone {
+		t.Errorf("%s: expected clusterIPs to be [%q]: %q", callName(before, after), api.ClusterIPNone, after.Spec.ClusterIPs)
+	}
+}
+
+func proveNodePortsAllocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+	t.Helper()
+
+	for _, p := range after.Spec.Ports {
+		if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+			t.Errorf("%s: expected nodePort to be allocated: %d", callName(before, after), p.NodePort)
+		}
+	}
+}
+
+func proveNodePortsDeallocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+	t.Helper()
+
+	if after != nil {
+		for _, p := range after.Spec.Ports {
+			if p.NodePort != 0 {
+				t.Errorf("%s: expected nodePort to be unset: %d", callName(before, after), p.NodePort)
+			}
+		}
+	}
+
+	if before != nil {
+		for _, p := range before.Spec.Ports {
+			if p.NodePort != 0 && portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+				t.Errorf("%s: expected nodePort to be deallocated: %d", callName(before, after), p.NodePort)
+			}
+		}
+	}
+}
+
+func proveHealthCheckNodePortAllocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+	t.Helper()
+
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, after.Spec.HealthCheckNodePort) {
+		t.Errorf("%s: expected healthCheckNodePort to be allocated: %d", callName(before, after), after.Spec.HealthCheckNodePort)
+	}
+}
+
+func proveHealthCheckNodePortDeallocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+	t.Helper()
+
+	if after != nil {
+		if after.Spec.HealthCheckNodePort != 0 {
+			t.Errorf("%s: expected healthCheckNodePort to be unset: %d", callName(before, after), after.Spec.HealthCheckNodePort)
+		}
+	}
+
+	if before != nil {
+		if before.Spec.HealthCheckNodePort != 0 && portIsAllocated(t, storage.alloc.serviceNodePorts, before.Spec.HealthCheckNodePort) {
+			t.Errorf("%s: expected healthCheckNodePort to be deallocated: %d", callName(before, after), before.Spec.HealthCheckNodePort)
+		}
+	}
+}
+
+type cudTestCase struct {
+	name   string
+	create svcTestCase
+	update svcTestCase
+}
+
+func helpTestCreateUpdateDelete(t *testing.T, testCases []cudTestCase) {
+	// NOTE: do not call t.Helper() here.  It's more useful for errors to be
+	// attributed to lines in this function than the caller of it.
+
+	// This test is ONLY with the gate enabled.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := genericapirequest.NewDefaultContext()
+
+			// Create the object as specified and check the results.
+			obj, err := storage.Create(ctx, tc.create.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if tc.create.expectError && err != nil {
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			defer storage.Delete(ctx, tc.create.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) // in case
+			if tc.create.expectError && err == nil {
+				t.Fatalf("unexpected success creating service")
+			}
+			createdSvc := obj.(*api.Service)
+			if !verifyEquiv(t, "create", &tc.create, createdSvc) {
+				return
+			}
+			verifyExpectations(t, storage, tc.create, tc.create.svc, createdSvc)
+
+			// Update the object to the new state and check the results.
+			obj, created, err := storage.Update(ctx, tc.update.svc.Name,
+				rest.DefaultUpdatedObjectInfo(tc.update.svc), rest.ValidateAllObjectFunc,
+				rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
+			if tc.update.expectError && err != nil {
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error updating service: %v", err)
+			}
+			if tc.update.expectError && err == nil {
+				t.Fatalf("unexpected success updating service")
+			}
+			if created {
+				t.Fatalf("unexpected create-on-update")
+			}
+			updatedSvc := obj.(*api.Service)
+			if !verifyEquiv(t, "update", &tc.update, updatedSvc) {
+				return
+			}
+			verifyExpectations(t, storage, tc.update, createdSvc, updatedSvc)
+
+			// Delete the object and check the results.
+			_, _, err = storage.Delete(ctx, tc.update.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error deleting service: %v", err)
+			}
+			verifyExpectations(t, storage, svcTestCase{ /* all false */ }, updatedSvc, nil)
+		})
+	}
+}
+
+type testingTInterface interface {
+	Helper()
+	Errorf(format string, args ...interface{})
+}
+
+type fakeTestingT struct {
+	t *testing.T
+}
+
+func (f fakeTestingT) Helper() {}
+
+func (f fakeTestingT) Errorf(format string, args ...interface{}) {
+	f.t.Logf(format, args...)
+}
+
+func verifyEquiv(t testingTInterface, call string, tc *svcTestCase, got *api.Service) bool {
+	t.Helper()
+
+	// For when we compare objects.
+	options := []cmp.Option{
+		// These are system-assigned values, we don't need to compare them.
+		cmpopts.IgnoreFields(api.Service{}, "UID", "ResourceVersion", "CreationTimestamp"),
+		// Treat nil slices and empty slices as the same (e.g. clusterIPs).
+		cmpopts.EquateEmpty(),
+	}
+
+	// For allocated fields, we want to be able to compare cleanly whether the
+	// input specified values or not.
+	want := tc.svc.DeepCopy()
+	if tc.expectClusterIPs || tc.expectHeadless {
+		if want.Spec.ClusterIP == "" {
+			want.Spec.ClusterIP = got.Spec.ClusterIP
+		}
+		if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+			if len(got.Spec.ClusterIPs) > len(want.Spec.ClusterIPs) {
+				want.Spec.ClusterIPs = append(want.Spec.ClusterIPs, got.Spec.ClusterIPs[len(want.Spec.ClusterIPs):]...)
+			}
+			if want.Spec.IPFamilyPolicy == nil {
+				want.Spec.IPFamilyPolicy = got.Spec.IPFamilyPolicy
+			}
+			if len(got.Spec.IPFamilies) > len(want.Spec.IPFamilies) {
+				want.Spec.IPFamilies = append(want.Spec.IPFamilies, got.Spec.IPFamilies[len(want.Spec.IPFamilies):]...)
+			}
+		}
+	}
+	if tc.expectNodePorts {
+		for i := range want.Spec.Ports {
+			p := &want.Spec.Ports[i]
+			if p.NodePort == 0 {
+				p.NodePort = got.Spec.Ports[i].NodePort
+			}
+		}
+	}
+	if tc.expectHealthCheckNodePort {
+		if want.Spec.HealthCheckNodePort == 0 {
+			want.Spec.HealthCheckNodePort = got.Spec.HealthCheckNodePort
+		}
+	}
+
+	if !cmp.Equal(want, got, options...) {
+		t.Errorf("unexpected result from %s:\n%s", call, cmp.Diff(want, got, options...))
+		return false
+	}
+	return true
+}
+
+// Quis custodiet ipsos custodes?
+func TestVerifyEquiv(t *testing.T) {
+	testCases := []struct {
+		name   string
+		input  svcTestCase
+		output *api.Service
+		expect bool
+	}{{
+		name: "ExternalName",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		expect: true,
+	}, {
+		name: "ClusterIPs_unspecified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: true,
+	}, {
+		name: "ClusterIPs_specified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: true,
+	}, {
+		name: "ClusterIPs_wrong",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.0", "2000:0")),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: false,
+	}, {
+		name: "ClusterIPs_partial",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: true,
+	}, {
+		name: "NodePort_unspecified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetUniqueNodePorts),
+		expect: true,
+	}, {
+		name: "NodePort_specified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
+		expect: true,
+	}, {
+		name: "NodePort_wrong",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(76)),
+		expect: false,
+	}, {
+		name: "NodePort_partial",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+				svctest.SetNodePorts(93)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetNodePorts(93, 76)),
+		expect: true,
+	}, {
+		name: "HealthCheckNodePort_unspecified",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(93)),
+		expect: true,
+	}, {
+		name: "HealthCheckNodePort_specified",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(93)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(93)),
+		expect: true,
+	}, {
+		name: "HealthCheckNodePort_wrong",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(93)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(76)),
+		expect: false,
+	}}
+
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := verifyEquiv(fakeTestingT{t}, "test", &tc.input, tc.output)
+			if result != tc.expect {
+				t.Errorf("expected %v, got %v", tc.expect, result)
+			}
+		})
+	}
+}
+
+func verifyExpectations(t *testing.T, storage *GenericREST, tc svcTestCase, before, after *api.Service) {
+	t.Helper()
+
+	if tc.expectClusterIPs {
+		proveClusterIPsAllocated(t, storage, before, after)
+	} else if tc.expectHeadless {
+		proveHeadless(t, storage, before, after)
+	} else {
+		proveClusterIPsDeallocated(t, storage, before, after)
+	}
+	if tc.expectNodePorts {
+		proveNodePortsAllocated(t, storage, before, after)
+	} else {
+		proveNodePortsDeallocated(t, storage, before, after)
+	}
+	if tc.expectHealthCheckNodePort {
+		proveHealthCheckNodePortAllocated(t, storage, before, after)
+	} else {
+		proveHealthCheckNodePortDeallocated(t, storage, before, after)
+	}
+}
+
+func TestFeatureExternalName(t *testing.T) {
+	testCases := []cudTestCase{{
+		name: "valid-valid",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName, svctest.SetExternalName("updated.example.com")),
+		},
+	}, {
+		name: "valid-blank",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		update: svcTestCase{
+			svc:         svctest.MakeService("foo", svctest.SetTypeExternalName, svctest.SetExternalName("")),
+			expectError: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
+func TestFeatureSelector(t *testing.T) {
+	testCases := []cudTestCase{{
+		name: "valid-valid",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				func(s *api.Service) {
+					s.Spec.Selector = map[string]string{"updated": "value"}
+				}),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "valid-nil",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				func(s *api.Service) {
+					s.Spec.Selector = nil
+				}),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "valid-empty",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				func(s *api.Service) {
+					s.Spec.Selector = map[string]string{}
+				}),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "nil-valid",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				func(s *api.Service) {
+					s.Spec.Selector = nil
+				}),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "empty-valid",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				func(s *api.Service) {
+					s.Spec.Selector = map[string]string{}
+				}),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
+func TestFeatureClusterIPs(t *testing.T) {
+	testCases := []cudTestCase{{
+		name: "clusterIP:valid-headless",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetHeadless),
+			expectError: true,
+		},
+	}, {
+		name: "clusterIP:headless-valid",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetHeadless),
+			expectHeadless: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetClusterIP("10.0.0.93")),
+			expectError: true,
+		},
+	}, {
+		name: "clusterIP:valid-valid",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetClusterIP("10.0.0.93")),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetClusterIP("10.0.0.76")),
+			expectError: true,
+		},
+	}, {
+		name: "clusterIPs:valid-valid",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetClusterIPs("10.0.0.93", "2000::93")),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetClusterIPs("10.0.0.76", "2000::76")),
+			expectError: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
+func TestFeaturePorts(t *testing.T) {
+	testCases := []cudTestCase{{
+		name: "add",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "remove",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "swap",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP),
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "modify",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("r", 53, intstr.FromInt(53), api.ProtocolTCP),
+					svctest.MakeServicePort("s", 53, intstr.FromInt(53), api.ProtocolUDP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "wipe",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts()),
+			expectError: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
+func TestFeatureSessionAffinity(t *testing.T) {
+	testCases := []cudTestCase{{
+		name: "None-ClientIPNoConfig",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSessionAffinity(api.ServiceAffinityNone)),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				func(s *api.Service) {
+					// Set it without setting the config
+					s.Spec.SessionAffinity = api.ServiceAffinityClientIP
+				}),
+			expectError: true,
+		},
+	}, {
+		name: "None-ClientIP",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSessionAffinity(api.ServiceAffinityNone)),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSessionAffinity(api.ServiceAffinityClientIP)),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "ClientIP-NoneWithConfig",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSessionAffinity(api.ServiceAffinityClientIP)),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSessionAffinity(api.ServiceAffinityClientIP),
+				func(s *api.Service) {
+					// Set it without wiping the config
+					s.Spec.SessionAffinity = api.ServiceAffinityNone
+				}),
+			expectError: true,
+		},
+	}, {
+		name: "ClientIP-None",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSessionAffinity(api.ServiceAffinityClientIP)),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSessionAffinity(api.ServiceAffinityNone),
+				func(s *api.Service) {
+					s.Spec.SessionAffinityConfig = nil
+				}),
+			expectClusterIPs: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
+func TestFeatureType(t *testing.T) {
+	testCases := []cudTestCase{{
+		name: "ExternalName-ClusterIP",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "ClusterIP-ExternalName",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+	}, {
+		name: "ExternalName-NodePort",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "NodePort-ExternalName",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+	}, {
+		name: "ExternalName-LoadBalancer",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "LoadBalancer-ExternalName",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+	}, {
+		name: "ClusterIP-NodePort",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "NodePort-ClusterIP",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "ClusterIP-LoadBalancer",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "LoadBalancer-ClusterIP",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "NodePort-LoadBalancer",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "LoadBalancer-NodePort",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "Headless-ExternalName",
+		create: svcTestCase{
+			svc:            svctest.MakeService("foo", svctest.SetHeadless),
+			expectHeadless: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+	}, {
+		name: "ExternalName-Headless",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		update: svcTestCase{
+			svc:            svctest.MakeService("foo", svctest.SetHeadless),
+			expectHeadless: true,
+		},
+	}, {
+		name: "Headless-NodePort",
+		create: svcTestCase{
+			svc:            svctest.MakeService("foo", svctest.SetHeadless),
+			expectHeadless: true,
+		},
+		update: svcTestCase{
+			svc:         svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectError: true,
+		},
+	}, {
+		name: "NodePort-Headless",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc:         svctest.MakeService("foo", svctest.SetHeadless),
+			expectError: true,
+		},
+	}, {
+		name: "Headless-LoadBalancer",
+		create: svcTestCase{
+			svc:            svctest.MakeService("foo", svctest.SetHeadless),
+			expectHeadless: true,
+		},
+		update: svcTestCase{
+			svc:         svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectError: true,
+		},
+	}, {
+		name: "LoadBalancer-Headless",
+		create: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc:         svctest.MakeService("foo", svctest.SetHeadless),
+			expectError: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
+// FIXME: externalIPs, lbip,
+// lbsourceranges, externalname, etp, hcnp, itp, PublishNotReadyAddresses,
+// ipfamilypolicy and list,
+// AllocateLoadBalancerNodePorts, LoadBalancerClass, status

From f4521aa75a12210d325001d2e3aef0972bda463d Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 1 Aug 2021 16:42:33 -0700
Subject: [PATCH 32/79] Fix validation on ETP: "" is not valid

This was causing tests to pass which ought not be passing.  This is not
an API change because we default the value of it when needed.  So we
would never see this in the wild, but it makes the tests sloppy.
---
 pkg/apis/core/validation/validation.go        | 76 +++++++++---------
 pkg/apis/core/validation/validation_test.go   | 78 ++++++++++++++++++-
 .../core/service/storage/rest_test.go         |  1 +
 3 files changed, 114 insertions(+), 41 deletions(-)

diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go
index 696563223b8..528fe50e267 100644
--- a/pkg/apis/core/validation/validation.go
+++ b/pkg/apis/core/validation/validation.go
@@ -4454,9 +4454,8 @@ func ValidateService(service *core.Service) field.ErrorList {
 	// validate LoadBalancerClass field
 	allErrs = append(allErrs, validateLoadBalancerClassField(nil, service)...)
 
-	// external traffic fields
-	allErrs = append(allErrs, validateServiceExternalTrafficFieldsValue(service)...)
-	allErrs = append(allErrs, validateServiceExternalTrafficFieldsCombination(service)...)
+	// external traffic policy fields
+	allErrs = append(allErrs, validateServiceExternalTrafficPolicy(service)...)
 
 	// internal traffic policy field
 	allErrs = append(allErrs, validateServiceInternalTrafficFieldsValue(service)...)
@@ -4508,22 +4507,46 @@ func validateServicePort(sp *core.ServicePort, requireName, isHeadlessService bo
 	return allErrs
 }
 
-// validateServiceExternalTrafficFieldsValue validates ExternalTraffic related annotations
-// have legal value.
-func validateServiceExternalTrafficFieldsValue(service *core.Service) field.ErrorList {
+func needsExternalTrafficPolicy(svc *api.Service) bool {
+	return svc.Spec.Type == core.ServiceTypeLoadBalancer || svc.Spec.Type == core.ServiceTypeNodePort
+}
+
+var validExternalTrafficPolicies = sets.NewString(
+	string(core.ServiceExternalTrafficPolicyTypeCluster),
+	string(core.ServiceExternalTrafficPolicyTypeLocal))
+
+func validateServiceExternalTrafficPolicy(service *core.Service) field.ErrorList {
 	allErrs := field.ErrorList{}
 
-	// Check first class fields.
-	if service.Spec.ExternalTrafficPolicy != "" &&
-		service.Spec.ExternalTrafficPolicy != core.ServiceExternalTrafficPolicyTypeCluster &&
-		service.Spec.ExternalTrafficPolicy != core.ServiceExternalTrafficPolicyTypeLocal {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("externalTrafficPolicy"), service.Spec.ExternalTrafficPolicy,
-			fmt.Sprintf("ExternalTrafficPolicy must be empty, %v or %v", core.ServiceExternalTrafficPolicyTypeCluster, core.ServiceExternalTrafficPolicyTypeLocal)))
+	fldPath := field.NewPath("spec")
+
+	if !needsExternalTrafficPolicy(service) {
+		if service.Spec.ExternalTrafficPolicy != "" {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("externalTrafficPolicy"), service.Spec.ExternalTrafficPolicy,
+				"may only be set when `type` is 'NodePort' or 'LoadBalancer'"))
+		}
+	} else {
+		if service.Spec.ExternalTrafficPolicy == "" {
+			allErrs = append(allErrs, field.Required(fldPath.Child("externalTrafficPolicy"), ""))
+		} else if !validExternalTrafficPolicies.Has(string(service.Spec.ExternalTrafficPolicy)) {
+			allErrs = append(allErrs, field.NotSupported(fldPath.Child("externalTrafficPolicy"),
+				service.Spec.ExternalTrafficPolicy, validExternalTrafficPolicies.List()))
+		}
 	}
 
-	if service.Spec.HealthCheckNodePort < 0 {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("healthCheckNodePort"), service.Spec.HealthCheckNodePort,
-			"HealthCheckNodePort must be not less than 0"))
+	if !apiservice.NeedsHealthCheck(service) {
+		if service.Spec.HealthCheckNodePort != 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("healthCheckNodePort"), service.Spec.HealthCheckNodePort,
+				"may only be set when `type` is 'LoadBalancer' and `externalTrafficPolicy` is 'Local'"))
+		}
+	} else {
+		if service.Spec.HealthCheckNodePort == 0 {
+			allErrs = append(allErrs, field.Required(fldPath.Child("healthCheckNodePort"), ""))
+		} else {
+			for _, msg := range validation.IsValidPortNum(int(service.Spec.HealthCheckNodePort)) {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("healthCheckNodePort"), service.Spec.HealthCheckNodePort, msg))
+			}
+		}
 	}
 
 	return allErrs
@@ -4559,29 +4582,6 @@ func validateServiceInternalTrafficFieldsValue(service *core.Service) field.Erro
 	return allErrs
 }
 
-// validateServiceExternalTrafficFieldsCombination validates if ExternalTrafficPolicy,
-// HealthCheckNodePort and Type combination are legal. For update, it should be called
-// after clearing externalTraffic related fields for the ease of transitioning between
-// different service types.
-func validateServiceExternalTrafficFieldsCombination(service *core.Service) field.ErrorList {
-	allErrs := field.ErrorList{}
-
-	if service.Spec.Type != core.ServiceTypeLoadBalancer &&
-		service.Spec.Type != core.ServiceTypeNodePort &&
-		service.Spec.ExternalTrafficPolicy != "" {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "externalTrafficPolicy"), service.Spec.ExternalTrafficPolicy,
-			"ExternalTrafficPolicy can only be set on NodePort and LoadBalancer service"))
-	}
-
-	if !apiservice.NeedsHealthCheck(service) &&
-		service.Spec.HealthCheckNodePort != 0 {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "healthCheckNodePort"), service.Spec.HealthCheckNodePort,
-			"HealthCheckNodePort can only be set on LoadBalancer service with ExternalTrafficPolicy=Local"))
-	}
-
-	return allErrs
-}
-
 // ValidateServiceCreate validates Services as they are created.
 func ValidateServiceCreate(service *core.Service) field.ErrorList {
 	return ValidateService(service)
diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go
index 64182ef6ef4..64cdf96bd7a 100644
--- a/pkg/apis/core/validation/validation_test.go
+++ b/pkg/apis/core/validation/validation_test.go
@@ -11003,6 +11003,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid load balancer protocol UDP 1",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports[0].Protocol = "UDP"
 			},
@@ -11012,6 +11013,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid load balancer protocol UDP 2",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports[0] = core.ServicePort{Name: "q", Port: 12345, Protocol: "UDP", TargetPort: intstr.FromInt(12345)}
 			},
@@ -11021,6 +11023,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "load balancer with mix protocol",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "UDP", TargetPort: intstr.FromInt(12345)})
 			},
@@ -11075,6 +11078,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type - loadbalancer",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 			},
 			numErrs: 0,
@@ -11083,6 +11087,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type - loadbalancer with allocateLoadBalancerNodePorts=false",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(false)
 			},
 			numErrs: 0,
@@ -11091,6 +11096,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "invalid type - missing AllocateLoadBalancerNodePorts for loadbalancer type",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 			},
 			numErrs: 1,
 		},
@@ -11098,6 +11104,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type loadbalancer 2 ports",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)})
 			},
@@ -11107,6 +11114,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid external load balancer 2 ports",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)})
 			},
@@ -11116,6 +11124,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "duplicate nodeports",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeNodePort
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)})
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "r", Port: 2, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(2)})
 			},
@@ -11125,6 +11134,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "duplicate nodeports (different protocols)",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeNodePort
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)})
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "r", Port: 2, Protocol: "UDP", NodePort: 1, TargetPort: intstr.FromInt(2)})
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "s", Port: 3, Protocol: "SCTP", NodePort: 1, TargetPort: intstr.FromInt(3)})
@@ -11161,6 +11171,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type - nodeport",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeNodePort
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 			},
 			numErrs: 0,
 		},
@@ -11168,6 +11179,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type - loadbalancer with allocateLoadBalancerNodePorts=true",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 			},
 			numErrs: 0,
@@ -11176,6 +11188,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type loadbalancer 2 ports",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)})
 			},
@@ -11185,6 +11198,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type loadbalancer with NodePort",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", NodePort: 12345, TargetPort: intstr.FromInt(12345)})
 			},
@@ -11194,6 +11208,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type=NodePort service with NodePort",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeNodePort
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", NodePort: 12345, TargetPort: intstr.FromInt(12345)})
 			},
 			numErrs: 0,
@@ -11202,6 +11217,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type=NodePort service without NodePort",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeNodePort
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)})
 			},
 			numErrs: 0,
@@ -11226,6 +11242,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "invalid public service with duplicate NodePort",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeNodePort
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "p1", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)})
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "p2", Port: 2, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(2)})
 			},
@@ -11235,6 +11252,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid type=LoadBalancer",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)})
 			},
@@ -11246,6 +11264,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "invalid port type=LoadBalancer",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "kubelet", Port: 10250, Protocol: "TCP", TargetPort: intstr.FromInt(12345)})
 			},
@@ -11255,6 +11274,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid LoadBalancer source range annotation",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Annotations[core.AnnotationLoadBalancerSourceRangesKey] = "1.2.3.4/8,  5.6.7.8/16"
 			},
@@ -11264,6 +11284,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "empty LoadBalancer source range annotation",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Annotations[core.AnnotationLoadBalancerSourceRangesKey] = ""
 			},
@@ -11280,6 +11301,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "invalid LoadBalancer source range annotation (invalid CIDR)",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Annotations[core.AnnotationLoadBalancerSourceRangesKey] = "1.2.3.4/33"
 			},
@@ -11296,6 +11318,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid LoadBalancer source range",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.LoadBalancerSourceRanges = []string{"1.2.3.4/8", "5.6.7.8/16"}
 			},
@@ -11305,6 +11328,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "empty LoadBalancer source range",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.LoadBalancerSourceRanges = []string{"   "}
 			},
@@ -11314,6 +11338,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "invalid LoadBalancer source range",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.LoadBalancerSourceRanges = []string{"foo.bar"}
 			},
@@ -11369,6 +11394,7 @@ func TestValidateServiceCreate(t *testing.T) {
 				s.Spec.ClusterIP = "None"
 				s.Spec.ClusterIPs = []string{"None"}
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 			},
 			numErrs: 1,
@@ -11377,6 +11403,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "invalid node port with clusterIP None",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeNodePort
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)})
 				s.Spec.ClusterIP = "None"
 				s.Spec.ClusterIPs = []string{"None"}
@@ -11463,6 +11490,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "sessionAffinityConfig can't be set when session affinity is None",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.SessionAffinity = core.ServiceAffinityNone
 				s.Spec.SessionAffinityConfig = &core.SessionAffinityConfig{
@@ -11916,6 +11944,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "valid LoadBalancerClass when type is LoadBalancer",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class")
 			},
@@ -11925,6 +11954,7 @@ func TestValidateServiceCreate(t *testing.T) {
 			name: "invalid LoadBalancerClass",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
+				s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				s.Spec.LoadBalancerClass = utilpointer.StringPtr("Bad/LoadBalancerClass")
 			},
@@ -11955,7 +11985,7 @@ func TestValidateServiceCreate(t *testing.T) {
 	}
 }
 
-func TestValidateServiceExternalTrafficFieldsCombination(t *testing.T) {
+func TestValidateServiceExternalTrafficPolicy(t *testing.T) {
 	testCases := []struct {
 		name     string
 		tweakSvc func(svc *core.Service) // Given a basic valid service, each test case can customize it.
@@ -12014,13 +12044,26 @@ func TestValidateServiceExternalTrafficFieldsCombination(t *testing.T) {
 			},
 			numErrs: 2,
 		},
+		{
+			name: "externalTrafficPolicy is required on NodePort service",
+			tweakSvc: func(s *core.Service) {
+				s.Spec.Type = core.ServiceTypeNodePort
+			},
+			numErrs: 1,
+		},
+		{
+			name: "externalTrafficPolicy is required on LoadBalancer service",
+			tweakSvc: func(s *core.Service) {
+				s.Spec.Type = core.ServiceTypeLoadBalancer
+			},
+			numErrs: 1,
+		},
 	}
 
 	for _, tc := range testCases {
 		svc := makeValidService()
 		tc.tweakSvc(&svc)
-		// TODO: This test is probably insufficient for such a big function under test.
-		errs := validateServiceExternalTrafficFieldsCombination(&svc)
+		errs := validateServiceExternalTrafficPolicy(&svc)
 		if len(errs) != tc.numErrs {
 			t.Errorf("Unexpected error list for case %q: %v", tc.name, errs.ToAggregate())
 		}
@@ -13511,6 +13554,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			name: "change type",
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 			},
 			numErrs: 0,
@@ -13526,6 +13570,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			name: "change type -> nodeport",
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 			},
 			numErrs: 0,
 		},
@@ -13535,6 +13580,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeLoadBalancer
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerSourceRanges = []string{"10.0.0.0/8"}
 			},
@@ -13547,6 +13593,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				oldSvc.Spec.LoadBalancerSourceRanges = []string{"10.0.0.0/8"}
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerSourceRanges = []string{"10.100.0.0/16"}
 			},
@@ -13558,6 +13605,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				newSvc.Spec.ClusterIP = "None"
 				newSvc.Spec.ClusterIPs = []string{"None"}
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 			},
 			numErrs: 1,
@@ -13631,6 +13679,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeClusterIP
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 
 				oldSvc.Spec.ClusterIP = "1.2.3.4"
 				oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"}
@@ -13645,6 +13694,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeClusterIP
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 
 				oldSvc.Spec.ClusterIP = ""
 				oldSvc.Spec.ClusterIPs = nil
@@ -13659,6 +13709,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeClusterIP
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 
 				oldSvc.Spec.ClusterIP = "1.2.3.4"
@@ -13674,6 +13725,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeClusterIP
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 
 				oldSvc.Spec.ClusterIP = ""
@@ -13690,6 +13742,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeLoadBalancer
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(false)
 			},
 			numErrs: 0,
@@ -13700,6 +13753,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeLoadBalancer
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(false)
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 			},
 			numErrs: 0,
@@ -13709,6 +13763,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeNodePort
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 
 				oldSvc.Spec.ClusterIP = "1.2.3.4"
 				oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"}
@@ -13723,6 +13778,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeNodePort
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 
 				oldSvc.Spec.ClusterIP = ""
 				oldSvc.Spec.ClusterIPs = nil
@@ -13765,6 +13821,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeNodePort
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 
 				oldSvc.Spec.ClusterIP = "1.2.3.4"
@@ -13780,6 +13837,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeNodePort
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 
 				oldSvc.Spec.ClusterIP = ""
@@ -13796,6 +13854,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeLoadBalancer
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 
 				oldSvc.Spec.ClusterIP = "1.2.3.4"
@@ -13812,6 +13871,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeLoadBalancer
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 
 				oldSvc.Spec.ClusterIP = ""
@@ -13858,6 +13918,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeLoadBalancer
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 
 				oldSvc.Spec.ClusterIP = "1.2.3.4"
 				oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"}
@@ -13873,6 +13934,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeLoadBalancer
 				oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 
 				oldSvc.Spec.ClusterIP = ""
 				oldSvc.Spec.ClusterIPs = nil
@@ -13915,6 +13977,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 			tweakSvc: func(oldSvc, newSvc *core.Service) {
 				oldSvc.Spec.Type = core.ServiceTypeNodePort
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 
 				oldSvc.Spec.Ports = append(oldSvc.Spec.Ports, core.ServicePort{Name: "q", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)})
 				newSvc.Spec.Ports = append(newSvc.Spec.Ports, core.ServicePort{Name: "q", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)})
@@ -14367,6 +14430,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old")
 
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old")
 			},
@@ -14380,6 +14444,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old")
 
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-new")
 			},
@@ -14393,6 +14458,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old")
 
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerClass = nil
 			},
@@ -14406,6 +14472,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.LoadBalancerClass = nil
 
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-new")
 			},
@@ -14417,6 +14484,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeClusterIP
 
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class")
 			},
@@ -14428,6 +14496,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeClusterIP
 
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerClass = nil
 			},
@@ -14439,6 +14508,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeClusterIP
 
 				newSvc.Spec.Type = core.ServiceTypeLoadBalancer
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 				newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("Bad/LoadBalancerclass")
 			},
@@ -14470,6 +14540,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.Type = core.ServiceTypeNodePort
 
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class")
 			},
 			numErrs: 2,
@@ -14506,6 +14577,7 @@ func TestValidateServiceUpdate(t *testing.T) {
 				oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class")
 
 				newSvc.Spec.Type = core.ServiceTypeNodePort
+				newSvc.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster
 				newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class")
 			},
 			numErrs: 2,
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index ae25c59c05c..affc06dbff1 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -597,6 +597,7 @@ func TestServiceRegistryUpdateLoadBalancerService(t *testing.T) {
 	// Modify to be loadbalancer.
 	svc2 := obj.(*api.Service).DeepCopy()
 	svc2.Spec.Type = api.ServiceTypeLoadBalancer
+	svc2.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeCluster
 	svc2.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
 	obj, _, err = storage.Update(ctx, svc2.Name, rest.DefaultUpdatedObjectInfo(svc2), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
 	if err != nil {

From 7b1e43665db2cc00e0156ee37f67a9962d7ab9f4 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 1 Aug 2021 17:08:05 -0700
Subject: [PATCH 33/79] Svc REST: Change ETP create test to a feature test

All the same test cases and more.
---
 .../core/service/storage/storage_test.go      | 541 +++++++++++-------
 1 file changed, 321 insertions(+), 220 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 2cba09b6fbb..38bea012a67 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5204,225 +5204,6 @@ func TestCreateInitNodePorts(t *testing.T) {
 	}
 }
 
-func TestCreateExternalTrafficPolicy(t *testing.T) {
-	testCases := []struct {
-		name        string
-		svc         *api.Service
-		expectError bool
-		expectHCNP  bool
-	}{{
-		name: "ExternalName_policy:none_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			svctest.SetExternalTrafficPolicy("")),
-		expectHCNP: false,
-	}, {
-		name: "ExternalName_policy:none_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			svctest.SetExternalTrafficPolicy(""),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "ExternalName_policy:Cluster_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
-		expectError: true,
-	}, {
-		name: "ExternalName_policy:Cluster_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "ExternalName_policy:Local_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-		expectError: true,
-	}, {
-		name: "ExternalName_policy:Local_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "ClusterIP_policy:none_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetExternalTrafficPolicy("")),
-		expectHCNP: false,
-	}, {
-		name: "ClusterIP_policy:none_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetExternalTrafficPolicy(""),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "ClusterIP_policy:Cluster_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
-		expectError: true,
-	}, {
-		name: "ClusterIP_policy:Cluster_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "ClusterIP_policy:Local_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-		expectError: true,
-	}, {
-		name: "ClusterIP_policy:Local_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "NodePort_policy:none_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetExternalTrafficPolicy("")),
-		expectHCNP: false,
-	}, {
-		name: "NodePort_policy:none_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetExternalTrafficPolicy(""),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "NodePort_policy:Cluster:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
-		expectHCNP: false,
-	}, {
-		name: "NodePort_policy:Cluster:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "NodePort_policy:Local_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-		expectHCNP: false,
-	}, {
-		name: "NodePort_policy:Local_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "LoadBalancer_policy:none_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy("")),
-		expectHCNP: false,
-	}, {
-		name: "LoadBalancer_policy:none_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(""),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "LoadBalancer_policy:Cluster_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
-		expectHCNP: false,
-	}, {
-		name: "LoadBalancer_policy:Cluster_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectError: true,
-	}, {
-		name: "LoadBalancer_policy:Local_hcnp:none",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-		expectHCNP: true,
-	}, {
-		name: "LoadBalancer_policy:Local_hcnp:specified",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(30000)),
-		expectHCNP: true,
-	}, {
-		name: "LoadBalancer_policy:Local_hcnp:negative",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(-1)),
-		expectError: true,
-	}}
-
-	// Do this in the outer scope for performance.
-	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	defer storage.Store.DestroyFunc()
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			ctx := genericapirequest.NewDefaultContext()
-			createdObj, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if tc.expectError && err != nil {
-				return
-			}
-			if err != nil {
-				t.Fatalf("unexpected error creating service: %v", err)
-			}
-			defer storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
-			if tc.expectError && err == nil {
-				t.Fatalf("unexpected success creating service")
-			}
-			createdSvc := createdObj.(*api.Service)
-
-			if !tc.expectHCNP {
-				if createdSvc.Spec.HealthCheckNodePort != 0 {
-					t.Fatalf("expected no HealthCheckNodePort, got %d", createdSvc.Spec.HealthCheckNodePort)
-				}
-				return
-			}
-
-			if createdSvc.Spec.HealthCheckNodePort == 0 {
-				t.Fatalf("expected a HealthCheckNodePort")
-			}
-			if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-				t.Errorf("expected HealthCheckNodePort to be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
-			}
-			if tc.svc.Spec.HealthCheckNodePort != 0 {
-				if want, got := tc.svc.Spec.HealthCheckNodePort, createdSvc.Spec.HealthCheckNodePort; want != got {
-					t.Errorf("wrong HealthCheckNodePort value: wanted %d, got %d", want, got)
-				}
-			}
-			for i, p := range createdSvc.Spec.Ports {
-				if p.NodePort == createdSvc.Spec.HealthCheckNodePort {
-					t.Errorf("HealthCheckNodePort overlaps NodePort[%d]", i)
-				}
-			}
-		})
-	}
-}
-
 // Prove that create skips allocations for Headless services.
 func TestCreateSkipsAllocationsForHeadless(t *testing.T) {
 	testCases := []struct {
@@ -6993,7 +6774,327 @@ func TestFeatureType(t *testing.T) {
 	helpTestCreateUpdateDelete(t, testCases)
 }
 
+func TestFeatureExternalTrafficPolicy(t *testing.T) {
+	testCases := []cudTestCase{{
+		name: "ExternalName_policy:none_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetExternalTrafficPolicy(""),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "ExternalName_policy:Cluster_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectError: true,
+		},
+	}, {
+		name: "ExternalName_policy:Cluster_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "ExternalName_policy:Local_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectError: true,
+		},
+	}, {
+		name: "ExternalName_policy:Local_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "ClusterIP_policy:none_hcnp:none_policy:Cluster_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetExternalTrafficPolicy("")),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectError: true,
+		},
+	}, {
+		name: "ClusterIP_policy:none_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetExternalTrafficPolicy(""),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "ClusterIP_policy:Cluster_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectError: true,
+		},
+	}, {
+		name: "ClusterIP_policy:Cluster_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "ClusterIP_policy:Local_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectError: true,
+		},
+	}, {
+		name: "ClusterIP_policy:Local_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "NodePort_policy:none_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy("")),
+			expectError: true,
+		},
+	}, {
+		name: "NodePort_policy:none_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy(""),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "NodePort_policy:Cluster_hcnp:none_policy:Local_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+	}, {
+		name: "NodePort_policy:Cluster_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "NodePort_policy:Local_hcnp:none_policy:Cluster_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+	}, {
+		name: "NodePort_policy:Local_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:none_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy("")),
+			expectError: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:none_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(""),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:Cluster_hcnp:none_policy:Local_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:Cluster_hcnp:none_policy:Local_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:Cluster_hcnp:specified",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectError: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:none_policy:Cluster_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:specified_policy:Cluster_hcnp:none",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeCluster)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: false,
+		},
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:specified_policy:Cluster_hcnp:different",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(30000)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(30001)),
+			expectError: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:none_policy:Inalid",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy("Invalid")),
+			expectError: true,
+		},
+	}, {
+		name: "LoadBalancer_policy:Local_hcnp:negative",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(-1)),
+			expectError: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
 // FIXME: externalIPs, lbip,
-// lbsourceranges, externalname, etp, hcnp, itp, PublishNotReadyAddresses,
+// lbsourceranges, externalname, itp, PublishNotReadyAddresses,
 // ipfamilypolicy and list,
 // AllocateLoadBalancerNodePorts, LoadBalancerClass, status

From 7cf75dbdd8f20b5d960a0fdf9cfb89f5edd5abb4 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Mon, 9 Aug 2021 22:14:15 -0700
Subject: [PATCH 34/79] Svc REST: Beef up NodePort tests

Remove old test from rest_test.go.
---
 .../core/service/storage/rest_test.go         | 115 -----------------
 .../core/service/storage/storage_test.go      | 121 +++++++++++++++++-
 2 files changed, 116 insertions(+), 120 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index affc06dbff1..5ca2ebee3ad 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -930,121 +930,6 @@ func TestServiceRegistryInternalTrafficPolicyLocalThenCluster(t *testing.T) {
 	}
 }
 
-func TestUpdateNodePorts(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	nodePortOp := portallocator.StartOperation(storage.alloc.serviceNodePorts, false)
-
-	testCases := []struct {
-		name                     string
-		oldService               *api.Service
-		newService               *api.Service
-		expectSpecifiedNodePorts []int
-	}{{
-		name: "Old service and new service have the same NodePort",
-		oldService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("", 6502, intstr.FromInt(6502), api.ProtocolTCP)),
-			svctest.SetNodePorts(30053)),
-		newService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("", 6502, intstr.FromInt(6502), api.ProtocolTCP)),
-			svctest.SetNodePorts(30053)),
-		expectSpecifiedNodePorts: []int{30053},
-	}, {
-		name: "Old service has more NodePorts than new service has",
-		oldService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30053, 30053)),
-		newService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP)),
-			svctest.SetNodePorts(30053)),
-		expectSpecifiedNodePorts: []int{30053},
-	}, {
-		name: "Change protocol of ServicePort without changing NodePort",
-		oldService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP)),
-			svctest.SetNodePorts(30053)),
-		newService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30053)),
-		expectSpecifiedNodePorts: []int{30053},
-	}, {
-		name: "Should allocate NodePort when changing service type to NodePort",
-		oldService: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetPorts(
-				svctest.MakeServicePort("", 6502, intstr.FromInt(6502), api.ProtocolUDP))),
-		newService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("", 6502, intstr.FromInt(6502), api.ProtocolUDP))),
-		expectSpecifiedNodePorts: []int{},
-	}, {
-		name: "Add new ServicePort with a different protocol without changing port numbers",
-		oldService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP)),
-			svctest.SetNodePorts(30053)),
-		newService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30053, 30053)),
-		expectSpecifiedNodePorts: []int{30053, 30053},
-	}, {
-		name: "Change service type from ClusterIP to NodePort with same NodePort number but different protocols",
-		oldService: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetPorts(
-				svctest.MakeServicePort("", 53, intstr.FromInt(6502), api.ProtocolTCP))),
-		newService: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("port-tcp", 53, intstr.FromInt(6502), api.ProtocolTCP),
-				svctest.MakeServicePort("port-udp", 53, intstr.FromInt(6502), api.ProtocolUDP)),
-			svctest.SetNodePorts(30053, 30053)),
-		expectSpecifiedNodePorts: []int{30053, 30053},
-	}}
-
-	for _, test := range testCases {
-		err := updateNodePorts(test.oldService, test.newService, nodePortOp)
-		if err != nil {
-			t.Errorf("%q: unexpected error: %v", test.name, err)
-			continue
-		}
-
-		serviceNodePorts := collectServiceNodePorts(test.newService)
-		if len(test.expectSpecifiedNodePorts) == 0 {
-			for _, nodePort := range serviceNodePorts {
-				if !storage.alloc.serviceNodePorts.Has(nodePort) {
-					t.Errorf("%q: unexpected NodePort %d, out of range", test.name, nodePort)
-				}
-			}
-		} else if !reflect.DeepEqual(serviceNodePorts, test.expectSpecifiedNodePorts) {
-			t.Errorf("%q: expected NodePorts %v, but got %v", test.name, test.expectSpecifiedNodePorts, serviceNodePorts)
-		}
-		for i := range serviceNodePorts {
-			nodePort := serviceNodePorts[i]
-			// Release the node port at the end of the test case.
-			storage.alloc.serviceNodePorts.Release(nodePort)
-		}
-	}
-}
-
 func TestServiceUpgrade(t *testing.T) {
 	requireDualStack := api.IPFamilyPolicyRequireDualStack
 
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 38bea012a67..b1f63dd20f2 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -6438,7 +6438,7 @@ func TestFeatureClusterIPs(t *testing.T) {
 
 func TestFeaturePorts(t *testing.T) {
 	testCases := []cudTestCase{{
-		name: "add",
+		name: "add_port",
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetPorts(
@@ -6453,7 +6453,39 @@ func TestFeaturePorts(t *testing.T) {
 			expectClusterIPs: true,
 		},
 	}, {
-		name: "remove",
+		name: "add_port_ClusterIP-NodePort",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "add_port_NodePort-ClusterIP",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "remove_port",
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetPorts(
@@ -6468,7 +6500,39 @@ func TestFeaturePorts(t *testing.T) {
 			expectClusterIPs: true,
 		},
 	}, {
-		name: "swap",
+		name: "remove_port_ClusterIP-NodePort",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+	}, {
+		name: "remove_port_NodePort-ClusterIP",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "swap_ports",
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetPorts(
@@ -6484,7 +6548,39 @@ func TestFeaturePorts(t *testing.T) {
 			expectClusterIPs: true,
 		},
 	}, {
-		name: "modify",
+		name: "modify_ports",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 8080, intstr.FromInt(8080), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 8443, intstr.FromInt(8443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "modify_protos",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolUDP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolUDP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "modify_ports_and_protos",
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetPorts(
@@ -6500,7 +6596,22 @@ func TestFeaturePorts(t *testing.T) {
 			expectClusterIPs: true,
 		},
 	}, {
-		name: "wipe",
+		name: "add_alt_proto",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP))),
+			expectClusterIPs: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP))),
+			expectClusterIPs: true,
+		},
+	}, {
+		name: "wipe_all",
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetPorts(

From 8bcba526b6ee7412848c136fc4a59df112c1569d Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 15 Aug 2021 12:31:31 -0700
Subject: [PATCH 35/79] Svc REST: Better errors on stack-downgrades

Converting dual-stack to single-stack needs good errors.
---
 pkg/apis/core/validation/validation.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go
index 528fe50e267..427339f9009 100644
--- a/pkg/apis/core/validation/validation.go
+++ b/pkg/apis/core/validation/validation.go
@@ -6470,7 +6470,7 @@ func validateUpgradeDowngradeClusterIPs(oldService, service *core.Service) field
 		// user *must* set IPFamilyPolicy == SingleStack
 		if len(service.Spec.ClusterIPs) == 1 {
 			if service.Spec.IPFamilyPolicy == nil || *(service.Spec.IPFamilyPolicy) != core.IPFamilyPolicySingleStack {
-				allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "`ipFamilyPolicy` must be set to 'SingleStack' when releasing the secondary clusterIP"))
+				allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "must be set to 'SingleStack' when releasing the secondary clusterIP"))
 			}
 		}
 	case len(oldService.Spec.ClusterIPs) < len(service.Spec.ClusterIPs):
@@ -6534,7 +6534,7 @@ func validateUpgradeDowngradeIPFamilies(oldService, service *core.Service) field
 		// user *must* set IPFamilyPolicy == SingleStack
 		if len(service.Spec.IPFamilies) == 1 {
 			if service.Spec.IPFamilyPolicy == nil || *(service.Spec.IPFamilyPolicy) != core.IPFamilyPolicySingleStack {
-				allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "`ipFamilyPolicy` must be set to 'SingleStack' when releasing the secondary ipFamily"))
+				allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "must be set to 'SingleStack' when releasing the secondary ipFamily"))
 			}
 		}
 	case len(oldService.Spec.IPFamilies) < len(service.Spec.IPFamilies):

From ced629e657a67da0d5d483a3283b404ee25f4a48 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 14 Aug 2021 10:49:17 -0700
Subject: [PATCH 36/79] Svc REST: Add proof funcs in feature test logic

Allows for more control of tests to assert specific things.
---
 pkg/registry/core/service/storage/storage_test.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index b1f63dd20f2..79dfeab5cfb 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5826,8 +5826,13 @@ type svcTestCase struct {
 	expectHeadless            bool
 	expectNodePorts           bool
 	expectHealthCheckNodePort bool
+
+	// Additional proofs, provided by the tests which use this.
+	prove []svcTestProof
 }
 
+type svcTestProof func(t *testing.T, storage *GenericREST, before, after *api.Service)
+
 func callName(before, after *api.Service) string {
 	if before == nil && after != nil {
 		return "create"
@@ -6287,6 +6292,10 @@ func verifyExpectations(t *testing.T, storage *GenericREST, tc svcTestCase, befo
 	} else {
 		proveHealthCheckNodePortDeallocated(t, storage, before, after)
 	}
+
+	for _, p := range tc.prove {
+		p(t, storage, before, after)
+	}
 }
 
 func TestFeatureExternalName(t *testing.T) {

From aea90a2324c2bf7ffe2c06f3c5426f94f09977f5 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 14 Aug 2021 10:54:23 -0700
Subject: [PATCH 37/79] Svc REST: add a beforeUpdate hook in feature tests

---
 pkg/registry/core/service/storage/storage_test.go | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 79dfeab5cfb..9a4a858e74a 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5991,9 +5991,10 @@ func proveHealthCheckNodePortDeallocated(t *testing.T, storage *GenericREST, bef
 }
 
 type cudTestCase struct {
-	name   string
-	create svcTestCase
-	update svcTestCase
+	name         string
+	create       svcTestCase
+	beforeUpdate func(t *testing.T, storage *GenericREST)
+	update       svcTestCase
 }
 
 func helpTestCreateUpdateDelete(t *testing.T, testCases []cudTestCase) {
@@ -6029,6 +6030,11 @@ func helpTestCreateUpdateDelete(t *testing.T, testCases []cudTestCase) {
 			}
 			verifyExpectations(t, storage, tc.create, tc.create.svc, createdSvc)
 
+			// Allow callers to do something between create and update.
+			if tc.beforeUpdate != nil {
+				tc.beforeUpdate(t, storage)
+			}
+
 			// Update the object to the new state and check the results.
 			obj, created, err := storage.Update(ctx, tc.update.svc.Name,
 				rest.DefaultUpdatedObjectInfo(tc.update.svc), rest.ValidateAllObjectFunc,

From 7887c4c8fc9f9846edee44ad8c3069a984a302a2 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 14 Aug 2021 11:00:49 -0700
Subject: [PATCH 38/79] Svc REST: allow tests to set cluster IP families

---
 pkg/registry/core/service/storage/storage_test.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 9a4a858e74a..f958b2e29ea 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5998,13 +5998,18 @@ type cudTestCase struct {
 }
 
 func helpTestCreateUpdateDelete(t *testing.T, testCases []cudTestCase) {
+	t.Helper()
+	helpTestCreateUpdateDeleteWithFamilies(t, testCases, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+}
+
+func helpTestCreateUpdateDeleteWithFamilies(t *testing.T, testCases []cudTestCase, ipFamilies []api.IPFamily) {
 	// NOTE: do not call t.Helper() here.  It's more useful for errors to be
 	// attributed to lines in this function than the caller of it.
 
 	// This test is ONLY with the gate enabled.
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
 
-	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	storage, _, server := newStorage(t, ipFamilies)
 	defer server.Terminate(t)
 	defer storage.Store.DestroyFunc()
 

From d1b83bad67f1a58fdda338481be922415c246cfb Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 5 Dec 2020 16:29:27 -0800
Subject: [PATCH 39/79] Svc REST: Move ResourceLocation() to 'inner' layer

Part of the de-layering effort.

Also move the test.
---
 pkg/registry/core/rest/storage_core.go        |   6 +-
 pkg/registry/core/service/storage/rest.go     |  73 +------
 .../core/service/storage/rest_test.go         | 158 +-------------
 pkg/registry/core/service/storage/storage.go  | 102 +++++++++-
 .../core/service/storage/storage_test.go      | 192 ++++++++++++++++--
 5 files changed, 293 insertions(+), 238 deletions(-)

diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go
index 61af3b7c0e8..443a955a8d8 100644
--- a/pkg/registry/core/rest/storage_core.go
+++ b/pkg/registry/core/rest/storage_core.go
@@ -261,12 +261,14 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		serviceIPAllocators[secondaryServiceClusterIPAllocator.IPFamily()] = secondaryServiceClusterIPAllocator
 	}
 
-	serviceRESTStorage, serviceStatusStorage, err := servicestore.NewGenericREST(
+	serviceRESTStorage, serviceStatusStorage, _, err := servicestore.NewGenericREST(
 		restOptionsGetter,
 		serviceClusterIPAllocator.IPFamily(),
 		serviceIPAllocators,
 		serviceNodePortAllocator,
-		endpointsStorage)
+		endpointsStorage,
+		podStorage.Pod,
+		c.ProxyTransport)
 	if err != nil {
 		return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err
 	}
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 8948207722c..30169021d08 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -19,17 +19,14 @@ package storage
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
-	"strconv"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	utilnet "k8s.io/apimachinery/pkg/util/net"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apimachinery/pkg/watch"
@@ -84,6 +81,7 @@ type ServiceStorage interface {
 	rest.Watcher
 	rest.StorageVersionProvider
 	rest.ResetFieldsStrategy
+	rest.Redirector
 }
 
 // NewREST returns a wrapper around the underlying generic storage and performs
@@ -360,74 +358,7 @@ var _ = rest.Redirector(&REST{})
 
 // ResourceLocation returns a URL to which one can send traffic for the specified service.
 func (rs *REST) ResourceLocation(ctx context.Context, id string) (*url.URL, http.RoundTripper, error) {
-	// Allow ID as "svcname", "svcname:port", or "scheme:svcname:port".
-	svcScheme, svcName, portStr, valid := utilnet.SplitSchemeNamePort(id)
-	if !valid {
-		return nil, nil, errors.NewBadRequest(fmt.Sprintf("invalid service request %q", id))
-	}
-
-	// If a port *number* was specified, find the corresponding service port name
-	if portNum, err := strconv.ParseInt(portStr, 10, 64); err == nil {
-		obj, err := rs.services.Get(ctx, svcName, &metav1.GetOptions{})
-		if err != nil {
-			return nil, nil, err
-		}
-		svc := obj.(*api.Service)
-		found := false
-		for _, svcPort := range svc.Spec.Ports {
-			if int64(svcPort.Port) == portNum {
-				// use the declared port's name
-				portStr = svcPort.Name
-				found = true
-				break
-			}
-		}
-		if !found {
-			return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no service port %d found for service %q", portNum, svcName))
-		}
-	}
-
-	obj, err := rs.endpoints.Get(ctx, svcName, &metav1.GetOptions{})
-	if err != nil {
-		return nil, nil, err
-	}
-	eps := obj.(*api.Endpoints)
-	if len(eps.Subsets) == 0 {
-		return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", svcName))
-	}
-	// Pick a random Subset to start searching from.
-	ssSeed := rand.Intn(len(eps.Subsets))
-	// Find a Subset that has the port.
-	for ssi := 0; ssi < len(eps.Subsets); ssi++ {
-		ss := &eps.Subsets[(ssSeed+ssi)%len(eps.Subsets)]
-		if len(ss.Addresses) == 0 {
-			continue
-		}
-		for i := range ss.Ports {
-			if ss.Ports[i].Name == portStr {
-				addrSeed := rand.Intn(len(ss.Addresses))
-				// This is a little wonky, but it's expensive to test for the presence of a Pod
-				// So we repeatedly try at random and validate it, this means that for an invalid
-				// service with a lot of endpoints we're going to potentially make a lot of calls,
-				// but in the expected case we'll only make one.
-				for try := 0; try < len(ss.Addresses); try++ {
-					addr := ss.Addresses[(addrSeed+try)%len(ss.Addresses)]
-					if err := isValidAddress(ctx, &addr, rs.pods); err != nil {
-						utilruntime.HandleError(fmt.Errorf("Address %v isn't valid (%v)", addr, err))
-						continue
-					}
-					ip := addr.IP
-					port := int(ss.Ports[i].Port)
-					return &url.URL{
-						Scheme: svcScheme,
-						Host:   net.JoinHostPort(ip, strconv.Itoa(port)),
-					}, rs.proxyTransport, nil
-				}
-				utilruntime.HandleError(fmt.Errorf("Failed to find a valid address, skipping subset: %v", ss))
-			}
-		}
-	}
-	return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", id))
+	return rs.services.ResourceLocation(ctx, id)
 }
 
 func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 5ca2ebee3ad..c79d7532aad 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
+	"net/url"
 	"reflect"
 	"sort"
 	"testing"
@@ -40,7 +42,6 @@ import (
 	"k8s.io/apiserver/pkg/util/dryrun"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
-	epstest "k8s.io/kubernetes/pkg/api/endpoints/testing"
 	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
@@ -178,14 +179,16 @@ func (s *serviceStorage) StorageVersion() runtime.GroupVersioner {
 
 // GetResetFields implements rest.ResetFieldsStrategy
 func (s *serviceStorage) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
+	//FIXME: should panic?
 	return nil
 }
 
-func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.EtcdTestServer) {
-	return NewTestRESTWithPods(t, nil, nil, ipFamilies)
+// ResourceLocation implements rest.Redirector
+func (s *serviceStorage) ResourceLocation(ctx context.Context, id string) (remoteLocation *url.URL, transport http.RoundTripper, err error) {
+	panic("not implemented")
 }
 
-func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Pod, ipFamilies []api.IPFamily) (*REST, *etcd3testing.EtcdTestServer) {
+func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 
 	podStorage, err := podstore.NewStorage(generic.RESTOptions{
@@ -197,13 +200,7 @@ func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Po
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
-	ctx := genericapirequest.NewDefaultContext()
-	for ix := range pods {
-		key, _ := podStorage.Pod.KeyFunc(ctx, pods[ix].Name)
-		if err := podStorage.Pod.Storage.Create(ctx, key, &pods[ix], nil, 0, false); err != nil {
-			t.Fatalf("Couldn't create pod: %v", err)
-		}
-	}
+
 	endpointStorage, err := endpointstore.NewREST(generic.RESTOptions{
 		StorageConfig:  etcdStorage.ForResource(schema.GroupResource{Resource: "endpoints"}),
 		Decorator:      generic.UndecoratedStorage,
@@ -212,12 +209,6 @@ func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Po
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
-	for ix := range endpoints {
-		key, _ := endpointStorage.KeyFunc(ctx, endpoints[ix].Name)
-		if err := endpointStorage.Store.Storage.Create(ctx, key, endpoints[ix], nil, 0, false); err != nil {
-			t.Fatalf("Couldn't create endpoint: %v", err)
-		}
-	}
 
 	var rPrimary ipallocator.Interface
 	var rSecondary ipallocator.Interface
@@ -281,7 +272,7 @@ func newInnerREST(t *testing.T, etcdStorage *storagebackend.ConfigForResource, i
 		ResourcePrefix: "endpoints",
 	})
 
-	inner, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, portAlloc, endpoints)
+	inner, _, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, portAlloc, endpoints, nil, nil)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
@@ -660,137 +651,6 @@ func makePod(name string, ips ...string) api.Pod {
 	return p
 }
 
-func TestServiceRegistryResourceLocation(t *testing.T) {
-	pods := []api.Pod{
-		makePod("unnamed", "1.2.3.4", "1.2.3.5"),
-		makePod("named", "1.2.3.6", "1.2.3.7"),
-		makePod("no-endpoints", "9.9.9.9"), // to prove this does not get chosen
-	}
-
-	endpoints := []*api.Endpoints{
-		epstest.MakeEndpoints("unnamed",
-			[]api.EndpointAddress{
-				epstest.MakeEndpointAddress("1.2.3.4", "unnamed"),
-			},
-			[]api.EndpointPort{
-				epstest.MakeEndpointPort("", 80),
-			}),
-		epstest.MakeEndpoints("unnamed2",
-			[]api.EndpointAddress{
-				epstest.MakeEndpointAddress("1.2.3.5", "unnamed"),
-			},
-			[]api.EndpointPort{
-				epstest.MakeEndpointPort("", 80),
-			}),
-		epstest.MakeEndpoints("named",
-			[]api.EndpointAddress{
-				epstest.MakeEndpointAddress("1.2.3.6", "named"),
-			},
-			[]api.EndpointPort{
-				epstest.MakeEndpointPort("p", 80),
-				epstest.MakeEndpointPort("q", 81),
-			}),
-		epstest.MakeEndpoints("no-endpoints", nil, nil), // to prove this does not get chosen
-	}
-
-	storage, server := NewTestRESTWithPods(t, endpoints, pods, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	ctx := genericapirequest.NewDefaultContext()
-	for _, name := range []string{"unnamed", "unnamed2", "no-endpoints"} {
-		_, err := storage.Create(ctx,
-			svctest.MakeService(name, svctest.SetPorts(
-				svctest.MakeServicePort("", 93, intstr.FromInt(80), api.ProtocolTCP))),
-			rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-		if err != nil {
-			t.Fatalf("unexpected error creating service %q: %v", name, err)
-		}
-
-	}
-	_, err := storage.Create(ctx,
-		svctest.MakeService("named", svctest.SetPorts(
-			svctest.MakeServicePort("p", 93, intstr.FromInt(80), api.ProtocolTCP),
-			svctest.MakeServicePort("q", 76, intstr.FromInt(81), api.ProtocolTCP))),
-		rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error creating service %q: %v", "named", err)
-	}
-	redirector := rest.Redirector(storage)
-
-	cases := []struct {
-		query  string
-		err    bool
-		expect string
-	}{{
-		query:  "unnamed",
-		expect: "//1.2.3.4:80",
-	}, {
-		query:  "unnamed:",
-		expect: "//1.2.3.4:80",
-	}, {
-		query:  "unnamed:93",
-		expect: "//1.2.3.4:80",
-	}, {
-		query:  "http:unnamed:",
-		expect: "http://1.2.3.4:80",
-	}, {
-		query:  "http:unnamed:93",
-		expect: "http://1.2.3.4:80",
-	}, {
-		query: "unnamed:80",
-		err:   true,
-	}, {
-		query:  "unnamed2",
-		expect: "//1.2.3.5:80",
-	}, {
-		query:  "named:p",
-		expect: "//1.2.3.6:80",
-	}, {
-		query:  "named:q",
-		expect: "//1.2.3.6:81",
-	}, {
-		query:  "named:93",
-		expect: "//1.2.3.6:80",
-	}, {
-		query:  "named:76",
-		expect: "//1.2.3.6:81",
-	}, {
-		query:  "http:named:p",
-		expect: "http://1.2.3.6:80",
-	}, {
-		query:  "http:named:q",
-		expect: "http://1.2.3.6:81",
-	}, {
-		query: "named:bad",
-		err:   true,
-	}, {
-		query: "no-endpoints",
-		err:   true,
-	}, {
-		query: "non-existent",
-		err:   true,
-	}}
-	for _, tc := range cases {
-		t.Run(tc.query, func(t *testing.T) {
-			location, _, err := redirector.ResourceLocation(ctx, tc.query)
-			if tc.err == false && err != nil {
-				t.Fatalf("unexpected error: %v", err)
-			}
-			if tc.err == true && err == nil {
-				t.Fatalf("unexpected success")
-			}
-			if !tc.err {
-				if location == nil {
-					t.Errorf("unexpected location: %v", location)
-				}
-				if e, a := tc.expect, location.String(); e != a {
-					t.Errorf("expected %q, but got %q", e, a)
-				}
-			}
-		})
-	}
-}
-
 func TestServiceRegistryList(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index a385fc7a96e..dbd01c4b097 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -18,16 +18,24 @@ package storage
 
 import (
 	"context"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
 
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/apiserver/pkg/util/dryrun"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/cri-api/pkg/errors"
 	"k8s.io/klog/v2"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
@@ -48,12 +56,18 @@ type EndpointsStorage interface {
 	rest.GracefulDeleter
 }
 
+type PodStorage interface {
+	rest.Getter
+}
+
 type GenericREST struct {
 	*genericregistry.Store
 	primaryIPFamily   api.IPFamily
 	secondaryIPFamily api.IPFamily
 	alloc             RESTAllocStuff
 	endpoints         EndpointsStorage
+	pods              PodStorage
+	proxyTransport    http.RoundTripper
 }
 
 // NewGenericREST returns a RESTStorage object that will work against services.
@@ -62,7 +76,9 @@ func NewGenericREST(
 	serviceIPFamily api.IPFamily,
 	ipAllocs map[api.IPFamily]ipallocator.Interface,
 	portAlloc portallocator.Interface,
-	endpoints EndpointsStorage) (*GenericREST, *StatusREST, error) {
+	endpoints EndpointsStorage,
+	pods PodStorage,
+	proxyTransport http.RoundTripper) (*GenericREST, *StatusREST, *svcreg.ProxyREST, error) {
 
 	strategy, _ := svcreg.StrategyForServiceCIDRs(ipAllocs[serviceIPFamily].CIDR(), len(ipAllocs) > 1)
 
@@ -81,7 +97,7 @@ func NewGenericREST(
 	}
 	options := &generic.StoreOptions{RESTOptions: optsGetter}
 	if err := store.CompleteWithOptions(options); err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	statusStore := *store
@@ -100,13 +116,15 @@ func NewGenericREST(
 		secondaryIPFamily: secondaryIPFamily,
 		alloc:             makeAlloc(serviceIPFamily, ipAllocs, portAlloc),
 		endpoints:         endpoints,
+		pods:              pods,
+		proxyTransport:    proxyTransport,
 	}
 	store.Decorator = genericStore.defaultOnRead
 	store.AfterDelete = genericStore.afterDelete
 	store.BeginCreate = genericStore.beginCreate
 	store.BeginUpdate = genericStore.beginUpdate
 
-	return genericStore, &StatusREST{store: &statusStore}, nil
+	return genericStore, &StatusREST{store: &statusStore}, &svcreg.ProxyREST{Redirector: genericStore, ProxyTransport: proxyTransport}, nil
 }
 
 // otherFamily returns the non-selected IPFamily.  This assumes the input is
@@ -352,3 +370,79 @@ func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Objec
 
 	return finish, nil
 }
+
+// Implement Redirector.
+var _ rest.Redirector = &GenericREST{}
+
+// ResourceLocation returns a URL to which one can send traffic for the specified service.
+func (r *GenericREST) ResourceLocation(ctx context.Context, id string) (*url.URL, http.RoundTripper, error) {
+	// Allow ID as "svcname", "svcname:port", or "scheme:svcname:port".
+	svcScheme, svcName, portStr, valid := utilnet.SplitSchemeNamePort(id)
+	if !valid {
+		return nil, nil, errors.NewBadRequest(fmt.Sprintf("invalid service request %q", id))
+	}
+
+	// If a port *number* was specified, find the corresponding service port name
+	if portNum, err := strconv.ParseInt(portStr, 10, 64); err == nil {
+		obj, err := r.Get(ctx, svcName, &metav1.GetOptions{})
+		if err != nil {
+			return nil, nil, err
+		}
+		svc := obj.(*api.Service)
+		found := false
+		for _, svcPort := range svc.Spec.Ports {
+			if int64(svcPort.Port) == portNum {
+				// use the declared port's name
+				portStr = svcPort.Name
+				found = true
+				break
+			}
+		}
+		if !found {
+			return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no service port %d found for service %q", portNum, svcName))
+		}
+	}
+
+	obj, err := r.endpoints.Get(ctx, svcName, &metav1.GetOptions{})
+	if err != nil {
+		return nil, nil, err
+	}
+	eps := obj.(*api.Endpoints)
+	if len(eps.Subsets) == 0 {
+		return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", svcName))
+	}
+	// Pick a random Subset to start searching from.
+	ssSeed := rand.Intn(len(eps.Subsets))
+	// Find a Subset that has the port.
+	for ssi := 0; ssi < len(eps.Subsets); ssi++ {
+		ss := &eps.Subsets[(ssSeed+ssi)%len(eps.Subsets)]
+		if len(ss.Addresses) == 0 {
+			continue
+		}
+		for i := range ss.Ports {
+			if ss.Ports[i].Name == portStr {
+				addrSeed := rand.Intn(len(ss.Addresses))
+				// This is a little wonky, but it's expensive to test for the presence of a Pod
+				// So we repeatedly try at random and validate it, this means that for an invalid
+				// service with a lot of endpoints we're going to potentially make a lot of calls,
+				// but in the expected case we'll only make one.
+				for try := 0; try < len(ss.Addresses); try++ {
+					addr := ss.Addresses[(addrSeed+try)%len(ss.Addresses)]
+					// TODO(thockin): do we really need this check?
+					if err := isValidAddress(ctx, &addr, r.pods); err != nil {
+						utilruntime.HandleError(fmt.Errorf("Address %v isn't valid (%v)", addr, err))
+						continue
+					}
+					ip := addr.IP
+					port := int(ss.Ports[i].Port)
+					return &url.URL{
+						Scheme: svcScheme,
+						Host:   net.JoinHostPort(ip, strconv.Itoa(port)),
+					}, r.proxyTransport, nil
+				}
+				utilruntime.HandleError(fmt.Errorf("Failed to find a valid address, skipping subset: %v", ss))
+			}
+		}
+	}
+	return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", id))
+}
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index f958b2e29ea..1b5247a7589 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -17,7 +17,6 @@ limitations under the License.
 package storage
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"reflect"
@@ -39,9 +38,12 @@ import (
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	epstest "k8s.io/kubernetes/pkg/api/endpoints/testing"
 	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
+	endpointstore "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
+	podstore "k8s.io/kubernetes/pkg/registry/core/pod/storage"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
@@ -64,16 +66,6 @@ func makePortAllocator(ports machineryutilnet.PortRange) portallocator.Interface
 	return al
 }
 
-type fakeEndpoints struct{}
-
-func (fakeEndpoints) Delete(_ context.Context, _ string, _ rest.ValidateObjectFunc, _ *metav1.DeleteOptions) (runtime.Object, bool, error) {
-	return nil, false, nil
-}
-
-func (fakeEndpoints) Get(_ context.Context, _ string, _ *metav1.GetOptions) (runtime.Object, error) {
-	return nil, nil
-}
-
 func ipIsAllocated(t *testing.T, alloc ipallocator.Interface, ipstr string) bool {
 	t.Helper()
 	ip := netutils.ParseIPSloppy(ipstr)
@@ -94,6 +86,10 @@ func portIsAllocated(t *testing.T, alloc portallocator.Interface, port int32) bo
 }
 
 func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
+	return newStorageWithPods(t, ipFamilies, nil, nil)
+}
+
+func newStorageWithPods(t *testing.T, ipFamilies []api.IPFamily, pods []api.Pod, endpoints []*api.Endpoints) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 	restOptions := generic.RESTOptions{
 		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
@@ -118,7 +114,45 @@ func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusR
 
 	portAlloc := makePortAllocator(*(machineryutilnet.ParsePortRangeOrDie("30000-32767")))
 
-	serviceStorage, statusStorage, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, portAlloc, fakeEndpoints{})
+	// Not all tests will specify pods and endpoints.
+	podStorage, err := podstore.NewStorage(generic.RESTOptions{
+		StorageConfig:           etcdStorage,
+		Decorator:               generic.UndecoratedStorage,
+		DeleteCollectionWorkers: 3,
+		ResourcePrefix:          "pods",
+	}, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected error from REST storage: %v", err)
+	}
+	if pods != nil && len(pods) > 0 {
+		ctx := genericapirequest.NewDefaultContext()
+		for ix := range pods {
+			key, _ := podStorage.Pod.KeyFunc(ctx, pods[ix].Name)
+			if err := podStorage.Pod.Storage.Create(ctx, key, &pods[ix], nil, 0, false); err != nil {
+				t.Fatalf("Couldn't create pod: %v", err)
+			}
+		}
+	}
+
+	endpointsStorage, err := endpointstore.NewREST(generic.RESTOptions{
+		StorageConfig:  etcdStorage,
+		Decorator:      generic.UndecoratedStorage,
+		ResourcePrefix: "endpoints",
+	})
+	if err != nil {
+		t.Fatalf("unexpected error from REST storage: %v", err)
+	}
+	if endpoints != nil && len(endpoints) > 0 {
+		ctx := genericapirequest.NewDefaultContext()
+		for ix := range endpoints {
+			key, _ := endpointsStorage.KeyFunc(ctx, endpoints[ix].Name)
+			if err := endpointsStorage.Store.Storage.Create(ctx, key, endpoints[ix], nil, 0, false); err != nil {
+				t.Fatalf("Couldn't create endpoint: %v", err)
+			}
+		}
+	}
+
+	serviceStorage, statusStorage, _, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, portAlloc, endpointsStorage, podStorage.Pod, nil)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
@@ -7229,3 +7263,137 @@ func TestFeatureExternalTrafficPolicy(t *testing.T) {
 // lbsourceranges, externalname, itp, PublishNotReadyAddresses,
 // ipfamilypolicy and list,
 // AllocateLoadBalancerNodePorts, LoadBalancerClass, status
+
+func TestServiceRegistryResourceLocation(t *testing.T) {
+	pods := []api.Pod{
+		makePod("unnamed", "1.2.3.4", "1.2.3.5"),
+		makePod("named", "1.2.3.6", "1.2.3.7"),
+		makePod("no-endpoints", "9.9.9.9"), // to prove this does not get chosen
+	}
+
+	endpoints := []*api.Endpoints{
+		epstest.MakeEndpoints("unnamed",
+			[]api.EndpointAddress{
+				epstest.MakeEndpointAddress("1.2.3.4", "unnamed"),
+			},
+			[]api.EndpointPort{
+				epstest.MakeEndpointPort("", 80),
+			}),
+		epstest.MakeEndpoints("unnamed2",
+			[]api.EndpointAddress{
+				epstest.MakeEndpointAddress("1.2.3.5", "unnamed"),
+			},
+			[]api.EndpointPort{
+				epstest.MakeEndpointPort("", 80),
+			}),
+		epstest.MakeEndpoints("named",
+			[]api.EndpointAddress{
+				epstest.MakeEndpointAddress("1.2.3.6", "named"),
+			},
+			[]api.EndpointPort{
+				epstest.MakeEndpointPort("p", 80),
+				epstest.MakeEndpointPort("q", 81),
+			}),
+		epstest.MakeEndpoints("no-endpoints", nil, nil), // to prove this does not get chosen
+	}
+
+	storage, _, server := newStorageWithPods(t, []api.IPFamily{api.IPv4Protocol}, pods, endpoints)
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	ctx := genericapirequest.NewDefaultContext()
+	for _, name := range []string{"unnamed", "unnamed2", "no-endpoints"} {
+		_, err := storage.Create(ctx,
+			svctest.MakeService(name,
+				svctest.SetPorts(
+					svctest.MakeServicePort("", 93, intstr.FromInt(80), api.ProtocolTCP))),
+			rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+		if err != nil {
+			t.Fatalf("unexpected error creating service %q: %v", name, err)
+		}
+
+	}
+	_, err := storage.Create(ctx,
+		svctest.MakeService("named",
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 93, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 76, intstr.FromInt(81), api.ProtocolTCP))),
+		rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error creating service %q: %v", "named", err)
+	}
+	redirector := rest.Redirector(storage)
+
+	cases := []struct {
+		query  string
+		err    bool
+		expect string
+	}{{
+		query:  "unnamed",
+		expect: "//1.2.3.4:80",
+	}, {
+		query:  "unnamed:",
+		expect: "//1.2.3.4:80",
+	}, {
+		query:  "unnamed:93",
+		expect: "//1.2.3.4:80",
+	}, {
+		query:  "http:unnamed:",
+		expect: "http://1.2.3.4:80",
+	}, {
+		query:  "http:unnamed:93",
+		expect: "http://1.2.3.4:80",
+	}, {
+		query: "unnamed:80",
+		err:   true,
+	}, {
+		query:  "unnamed2",
+		expect: "//1.2.3.5:80",
+	}, {
+		query:  "named:p",
+		expect: "//1.2.3.6:80",
+	}, {
+		query:  "named:q",
+		expect: "//1.2.3.6:81",
+	}, {
+		query:  "named:93",
+		expect: "//1.2.3.6:80",
+	}, {
+		query:  "named:76",
+		expect: "//1.2.3.6:81",
+	}, {
+		query:  "http:named:p",
+		expect: "http://1.2.3.6:80",
+	}, {
+		query:  "http:named:q",
+		expect: "http://1.2.3.6:81",
+	}, {
+		query: "named:bad",
+		err:   true,
+	}, {
+		query: "no-endpoints",
+		err:   true,
+	}, {
+		query: "non-existent",
+		err:   true,
+	}}
+	for _, tc := range cases {
+		t.Run(tc.query, func(t *testing.T) {
+			location, _, err := redirector.ResourceLocation(ctx, tc.query)
+			if tc.err == false && err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tc.err == true && err == nil {
+				t.Fatalf("unexpected success")
+			}
+			if !tc.err {
+				if location == nil {
+					t.Errorf("unexpected location: %v", location)
+				}
+				if e, a := tc.expect, location.String(); e != a {
+					t.Errorf("expected %q, but got %q", e, a)
+				}
+			}
+		})
+	}
+}

From 7602260d0a85f2f987604ea13ce68c6ac3b68cb9 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 22 Aug 2021 21:38:20 -0700
Subject: [PATCH 40/79] Svc REST: Fix comments to make next commits easier

---
 pkg/registry/core/service/storage/rest.go | 60 +++++++++++++----------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 30169021d08..45edb26b085 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -560,14 +560,14 @@ func (al *RESTAllocStuff) allocUpdateServiceClusterIPsNew(service *api.Service,
 // executed successfully
 func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Service, service *api.Service, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
 
-	// We don't want to upgrade (add an IP) or downgrade (remove an IP)
-	// following a cluster downgrade/upgrade to/from dual-stackness
-	// a PreferDualStack service following principle of least surprise
-	// That means: PreferDualStack service will only be upgraded
-	// if:
-	// - changes type to RequireDualStack
-	// - manually adding or removing ClusterIP (secondary)
-	// - manually adding or removing IPFamily (secondary)
+	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
+	// PreferDualStack services following a cluster change to/from
+	// dual-stackness.
+	//
+	// That means a PreferDualStack service will only be upgraded/downgraded
+	// when:
+	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
+	// - adding or removing a secondary clusterIP or ipFamily
 	if isMatchingPreferDualStackClusterIPFields(oldService, service) {
 		return allocated, toRelease, nil // nothing more to do.
 	}
@@ -765,14 +765,14 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		return nil
 	}
 
-	// We don't want to upgrade (add an IP) or downgrade (remove an IP)
-	// following a cluster downgrade/upgrade to/from dual-stackness
-	// a PreferDualStack service following principle of least surprise
-	// That means: PreferDualStack service will only be upgraded
-	// if:
-	// - changes type to RequireDualStack
-	// - manually adding or removing ClusterIP (secondary)
-	// - manually adding or removing IPFamily (secondary)
+	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
+	// PreferDualStack services following a cluster change to/from
+	// dual-stackness.
+	//
+	// That means a PreferDualStack service will only be upgraded/downgraded
+	// when:
+	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
+	// - adding or removing a secondary clusterIP or ipFamily
 	if isMatchingPreferDualStackClusterIPFields(oldService, service) {
 		return nil // nothing more to do.
 	}
@@ -794,7 +794,8 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		}
 	}
 
-	// Infer IPFamilies[] from ClusterIPs[].
+	// Infer IPFamilies[] from ClusterIPs[].  Further checks happen below,
+	// after the special cases.
 	for i, ip := range service.Spec.ClusterIPs {
 		if ip == api.ClusterIPNone {
 			break
@@ -804,7 +805,7 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		// so the following is safe to do
 		isIPv6 := netutils.IsIPv6String(ip)
 
-		// Family is not specified yet.
+		// If the corresponding family is not specified, add it.
 		if i >= len(service.Spec.IPFamilies) {
 			if isIPv6 {
 				// first make sure that family(ip) is configured
@@ -825,13 +826,15 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 	}
 
 	// Infer IPFamilyPolicy from IPFamilies[].  This block does not handle the
-	// final defaulting - that happens a bit later, after special-cases.
+	// final defaulting - that happens a bit later, after special cases.
 	if service.Spec.IPFamilyPolicy == nil && len(service.Spec.IPFamilies) == 2 {
 		requireDualStack := api.IPFamilyPolicyRequireDualStack
 		service.Spec.IPFamilyPolicy = &requireDualStack
 	}
 
-	// Special-case: headless+selectorless
+	// Special-case: headless + selectorless.  This has to happen before other
+	// checks because it explicitly allows combinations of inputs that would
+	// otherwise be errors.
 	if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone && len(service.Spec.Selector) == 0 {
 		// If the use said nothing about policy and we can't infer it, they get dual-stack
 		if service.Spec.IPFamilyPolicy == nil {
@@ -863,24 +866,27 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		return nil
 	}
 
+	//
+	// Everything below this MUST happen *after* the above special cases.
+	//
+
 	// ipfamily check
 	// the following applies on all type of services including headless w/ selector
 	el := make(field.ErrorList, 0)
 
-	// asking for dual stack on a non dual stack cluster
-	// should fail without assigning any family
+	// Demanding dual-stack on a non dual-stack cluster.
 	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack && len(al.serviceIPAllocatorsByFamily) < 2 {
 		el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "Cluster is not configured for dual stack services"))
 	}
 
-	// if there is a family requested then it has to be configured on cluster
+	// If there is a family requested then it has to be configured on cluster.
 	for i, ipFamily := range service.Spec.IPFamilies {
 		if _, found := al.serviceIPAllocatorsByFamily[ipFamily]; !found {
 			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilies").Index(i), service.Spec.ClusterIPs, fmt.Sprintf("ipfamily %v is not configured on cluster", ipFamily)))
 		}
 	}
 
-	// if we have validation errors return them and bail out
+	// If we have validation errors, don't bother with the rest.
 	if len(el) > 0 {
 		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
 	}
@@ -892,13 +898,13 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		service.Spec.IPFamilyPolicy = &singleStack
 	}
 
-	// nil families, gets cluster default (if feature flag is not in effect, the strategy will take care of removing it)
+	// nil families, gets cluster default
 	if len(service.Spec.IPFamilies) == 0 {
 		service.Spec.IPFamilies = []api.IPFamily{al.defaultServiceIPFamily}
 	}
 
-	// is this service looking for dual stack, and this cluster does have two families?
-	// if so, then append the missing family
+	// If this service is looking for dual-stack and this cluster does have two
+	// families, append the missing family.
 	if *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack &&
 		len(service.Spec.IPFamilies) == 1 &&
 		len(al.serviceIPAllocatorsByFamily) == 2 {

From 650f8cfd35123e14e0d42510bd51814bdfaa356d Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Mon, 16 Aug 2021 08:52:48 -0700
Subject: [PATCH 41/79] Svc REST: Validate input before IP allocation

This commit started as removing FIXME comments, but in doing so I
realized that the IP allocation process was using unvalidated user
input.  Before de-layering, validation was called twice - once before
init and once after, which the init code depended on.

Fortunately (or not?) we had duplicative checks that caught errors but
with less friendly messages.

This commit calls validation before initializing the rest of the
IP-related fields.

This also re-organizes that code a bit, cleans up error messages and
comments, and adds a test SPECIFICALLY for the errors in those cases.
---
 pkg/apis/core/validation/validation.go        |  12 +-
 pkg/registry/core/service/storage/rest.go     | 176 ++++++++++++++----
 .../core/service/storage/storage_test.go      | 146 ++++++++++++++-
 3 files changed, 282 insertions(+), 52 deletions(-)

diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go
index 427339f9009..1e1ffc3850c 100644
--- a/pkg/apis/core/validation/validation.go
+++ b/pkg/apis/core/validation/validation.go
@@ -4358,7 +4358,7 @@ func ValidateService(service *core.Service) field.ErrorList {
 	}
 
 	// dualstack <-> ClusterIPs <-> ipfamilies
-	allErrs = append(allErrs, validateServiceClusterIPsRelatedFields(service)...)
+	allErrs = append(allErrs, ValidateServiceClusterIPsRelatedFields(service)...)
 
 	ipPath := specPath.Child("externalIPs")
 	for i, ip := range service.Spec.ExternalIPs {
@@ -6305,8 +6305,10 @@ func ValidateSpreadConstraintNotRepeat(fldPath *field.Path, constraint core.Topo
 	return nil
 }
 
-// validateServiceClusterIPsRelatedFields validates .spec.ClusterIPs,, .spec.IPFamilies, .spec.ipFamilyPolicy
-func validateServiceClusterIPsRelatedFields(service *core.Service) field.ErrorList {
+// ValidateServiceClusterIPsRelatedFields validates .spec.ClusterIPs,,
+// .spec.IPFamilies, .spec.ipFamilyPolicy.  This is exported because it is used
+// during IP init and allocation.
+func ValidateServiceClusterIPsRelatedFields(service *core.Service) field.ErrorList {
 	// ClusterIP, ClusterIPs, IPFamilyPolicy and IPFamilies are validated prior (all must be unset) for ExternalName service
 	if service.Spec.Type == core.ServiceTypeExternalName {
 		return field.ErrorList{}
@@ -6328,12 +6330,12 @@ func validateServiceClusterIPsRelatedFields(service *core.Service) field.ErrorLi
 		if len(service.Spec.ClusterIPs) == 0 {
 			allErrs = append(allErrs, field.Required(clusterIPsField, ""))
 		} else if service.Spec.ClusterIPs[0] != service.Spec.ClusterIP {
-			allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "element [0] must match clusterIP"))
+			allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "first value must match `clusterIP`"))
 		}
 	} else { // ClusterIP == ""
 		// If ClusterIP is not set, ClusterIPs must also be unset.
 		if len(service.Spec.ClusterIPs) != 0 {
-			allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "must be empty when clusterIP is empty"))
+			allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "must be empty when `clusterIP` is not specified"))
 		}
 	}
 
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 45edb26b085..6e8ea1d3f33 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -19,7 +19,6 @@ package storage
 import (
 	"context"
 	"fmt"
-	"net"
 	"net/http"
 	"net/url"
 
@@ -36,6 +35,7 @@ import (
 	"k8s.io/klog/v2"
 	apiservice "k8s.io/kubernetes/pkg/api/service"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/features"
 	registry "k8s.io/kubernetes/pkg/registry/core/service"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
@@ -751,6 +751,68 @@ func isMatchingPreferDualStackClusterIPFields(oldService, service *api.Service)
 	return true
 }
 
+func sameClusterIPs(lhs, rhs *api.Service) bool {
+	if len(rhs.Spec.ClusterIPs) != len(lhs.Spec.ClusterIPs) {
+		return false
+	}
+
+	for i, ip := range rhs.Spec.ClusterIPs {
+		if lhs.Spec.ClusterIPs[i] != ip {
+			return false
+		}
+	}
+
+	return true
+}
+
+func reducedClusterIPs(before, after *api.Service) bool {
+	if len(after.Spec.ClusterIPs) == 0 { // Not specified
+		return false
+	}
+	return len(after.Spec.ClusterIPs) < len(before.Spec.ClusterIPs)
+}
+
+func sameIPFamilies(lhs, rhs *api.Service) bool {
+	if len(rhs.Spec.IPFamilies) != len(lhs.Spec.IPFamilies) {
+		return false
+	}
+
+	for i, family := range rhs.Spec.IPFamilies {
+		if lhs.Spec.IPFamilies[i] != family {
+			return false
+		}
+	}
+
+	return true
+}
+
+func reducedIPFamilies(before, after *api.Service) bool {
+	if len(after.Spec.IPFamilies) == 0 { // Not specified
+		return false
+	}
+	return len(after.Spec.IPFamilies) < len(before.Spec.IPFamilies)
+}
+
+// Helper to get the IP family of a given IP.
+func familyOf(ip string) api.IPFamily {
+	if netutils.IsIPv4String(ip) {
+		return api.IPv4Protocol
+	}
+	if netutils.IsIPv6String(ip) {
+		return api.IPv6Protocol
+	}
+	return api.IPFamily("unknown")
+}
+
+// Helper to avoid nil-checks all over.  Callers of this need to be checking
+// for an exact value.
+func getIPFamilyPolicy(svc *api.Service) api.IPFamilyPolicyType {
+	if svc.Spec.IPFamilyPolicy == nil {
+		return "" // callers need to handle this
+	}
+	return *svc.Spec.IPFamilyPolicy
+}
+
 // attempts to default service ip families according to cluster configuration
 // while ensuring that provided families are configured on cluster.
 func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) error {
@@ -777,20 +839,63 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		return nil // nothing more to do.
 	}
 
-	// two families or two IPs with SingleStack
-	if service.Spec.IPFamilyPolicy != nil {
-		el := make(field.ErrorList, 0)
-		if *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicySingleStack {
-			if len(service.Spec.ClusterIPs) == 2 {
-				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "must be RequireDualStack or PreferDualStack when multiple 'clusterIPs' are specified"))
+	// Update-only prep work.
+	if oldService != nil {
+		np := service.Spec.IPFamilyPolicy
+
+		// If they didn't specify policy, or specified anything but
+		// single-stack AND they reduced these fields, it's an error.  They
+		// need to specify policy.
+		if np == nil || *np != api.IPFamilyPolicySingleStack {
+			el := make(field.ErrorList, 0)
+
+			if reducedClusterIPs(oldService, service) {
+				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
+					"must be 'SingleStack' to release the secondary cluster IP"))
 			}
-			if len(service.Spec.IPFamilies) == 2 {
-				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "must be RequireDualStack or PreferDualStack when multiple 'ipFamilies' are specified"))
+			if reducedIPFamilies(oldService, service) {
+				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
+					"must be 'SingleStack' to release the secondary IP family"))
+			}
+
+			if len(el) > 0 {
+				return errors.NewInvalid(api.Kind("Service"), service.Name, el)
+			}
+		} else { // policy must be SingleStack
+			// Update: As long as ClusterIPs and IPFamilies have not changed,
+			// setting policy to single-stack is clear intent.
+			if *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicySingleStack {
+				if sameClusterIPs(oldService, service) && len(service.Spec.ClusterIPs) > 1 {
+					service.Spec.ClusterIPs = service.Spec.ClusterIPs[0:1]
+				}
+				if sameIPFamilies(oldService, service) && len(service.Spec.IPFamilies) > 1 {
+					service.Spec.IPFamilies = service.Spec.IPFamilies[0:1]
+				}
 			}
 		}
+	}
 
-		if len(el) > 0 {
-			return errors.NewInvalid(api.Kind("Service"), service.Name, el)
+	// Do some loose pre-validation of the input.  This makes it easier in the
+	// rest of allocation code to not have to consider corner cases.
+	// TODO(thockin): when we tighten validation (e.g. to require IPs) we will
+	// need a "strict" and a "loose" form of this.
+	if el := validation.ValidateServiceClusterIPsRelatedFields(service); len(el) != 0 {
+		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
+	}
+
+	//TODO(thockin): Move this logic to validation?
+	el := make(field.ErrorList, 0)
+
+	// Make sure ipFamilyPolicy makes sense for the provided ipFamilies and
+	// clusterIPs.  Further checks happen below - after the special cases.
+	if getIPFamilyPolicy(service) == api.IPFamilyPolicySingleStack {
+		if len(service.Spec.ClusterIPs) == 2 {
+			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
+				"must be 'RequireDualStack' or 'PreferDualStack' when multiple cluster IPs are specified"))
+		}
+		if len(service.Spec.IPFamilies) == 2 {
+			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
+				"must be 'RequireDualStack' or 'PreferDualStack' when multiple IP families are specified"))
 		}
 	}
 
@@ -801,30 +906,30 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 			break
 		}
 
-		// we have previously validated for ip correctness and if family exist it will match ip family
-		// so the following is safe to do
-		isIPv6 := netutils.IsIPv6String(ip)
+		// We previously validated that IPs are well-formed and that if an
+		// ipFamilies[] entry exists it matches the IP.
+		fam := familyOf(ip)
 
 		// If the corresponding family is not specified, add it.
 		if i >= len(service.Spec.IPFamilies) {
-			if isIPv6 {
-				// first make sure that family(ip) is configured
-				if _, found := al.serviceIPAllocatorsByFamily[api.IPv6Protocol]; !found {
-					el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv6 on a cluster which is not configured for it")}
-					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
-				}
-				service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
+			// Families are checked more later, but this is a better error in
+			// this specific case (indicating the user-provided IP, rather
+			// than than the auto-assigned family).
+			if _, found := al.serviceIPAllocatorsByFamily[fam]; !found {
+				el = append(el, field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs,
+					fmt.Sprintf("%s is not configured on this cluster", fam)))
 			} else {
-				// first make sure that family(ip) is configured
-				if _, found := al.serviceIPAllocatorsByFamily[api.IPv4Protocol]; !found {
-					el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv4 on a cluster which is not configured for it")}
-					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
-				}
-				service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol)
+				// OK to infer.
+				service.Spec.IPFamilies = append(service.Spec.IPFamilies, fam)
 			}
 		}
 	}
 
+	// If we have validation errors, bail out now so we don't make them worse.
+	if len(el) > 0 {
+		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
+	}
+
 	// Infer IPFamilyPolicy from IPFamilies[].  This block does not handle the
 	// final defaulting - that happens a bit later, after special cases.
 	if service.Spec.IPFamilyPolicy == nil && len(service.Spec.IPFamilies) == 2 {
@@ -870,19 +975,18 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 	// Everything below this MUST happen *after* the above special cases.
 	//
 
-	// ipfamily check
-	// the following applies on all type of services including headless w/ selector
-	el := make(field.ErrorList, 0)
-
 	// Demanding dual-stack on a non dual-stack cluster.
-	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack && len(al.serviceIPAllocatorsByFamily) < 2 {
-		el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "Cluster is not configured for dual stack services"))
+	if getIPFamilyPolicy(service) == api.IPFamilyPolicyRequireDualStack {
+		if len(al.serviceIPAllocatorsByFamily) < 2 {
+			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
+				"this cluster is not configured for dual-stack services"))
+		}
 	}
 
 	// If there is a family requested then it has to be configured on cluster.
 	for i, ipFamily := range service.Spec.IPFamilies {
 		if _, found := al.serviceIPAllocatorsByFamily[ipFamily]; !found {
-			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilies").Index(i), service.Spec.ClusterIPs, fmt.Sprintf("ipfamily %v is not configured on cluster", ipFamily)))
+			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilies").Index(i), ipFamily, "not configured on this cluster"))
 		}
 	}
 
@@ -911,9 +1015,7 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 
 		if service.Spec.IPFamilies[0] == api.IPv4Protocol {
 			service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
-		}
-
-		if service.Spec.IPFamilies[0] == api.IPv6Protocol {
+		} else if service.Spec.IPFamilies[0] == api.IPv6Protocol {
 			service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol)
 		}
 	}
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 1b5247a7589..16e9d6593ec 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -376,16 +377,6 @@ func fmtIPFamilies(fams []api.IPFamily) string {
 	return fmt.Sprintf("%v", fams)
 }
 
-func familyOf(ip string) api.IPFamily {
-	if netutils.IsIPv4String(ip) {
-		return api.IPv4Protocol
-	}
-	if netutils.IsIPv6String(ip) {
-		return api.IPv6Protocol
-	}
-	return api.IPFamily("unknown")
-}
-
 // Prove that create ignores IP and IPFamily stuff when type is ExternalName.
 func TestCreateIgnoresIPsForExternalName(t *testing.T) {
 	type testCase struct {
@@ -4765,6 +4756,141 @@ func TestCreateInitIPFields(t *testing.T) {
 	}
 }
 
+// There are enough corner-cases that it's useful to have a test that asserts
+// the errors.  Some of these are in other tests, but this is clearer.
+func TestCreateInvalidClusterIPInputs(t *testing.T) {
+	testCases := []struct {
+		name     string
+		families []api.IPFamily
+		svc      *api.Service
+		expect   []string
+	}{{
+		name:     "bad_ipFamilyPolicy",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyType("garbage"))),
+		expect: []string{"Unsupported value"},
+	}, {
+		name:     "requiredual_ipFamilyPolicy_on_singlestack",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+		expect: []string{"cluster is not configured for dual-stack"},
+	}, {
+		name:     "bad_ipFamilies_0_value",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPFamily("garbage"))),
+		expect: []string{"Unsupported value"},
+	}, {
+		name:     "bad_ipFamilies_1_value",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPv4Protocol, api.IPFamily("garbage"))),
+		expect: []string{"Unsupported value"},
+	}, {
+		name:     "bad_ipFamilies_2_value",
+		families: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol, api.IPFamily("garbage"))),
+		expect: []string{"Unsupported value"},
+	}, {
+		name:     "wrong_ipFamily",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPv6Protocol)),
+		expect: []string{"not configured on this cluster"},
+	}, {
+		name:     "too_many_ipFamilies_on_singlestack",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+		expect: []string{"not configured on this cluster"},
+	}, {
+		name:     "dup_ipFamily_singlestack",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv4Protocol)),
+		expect: []string{"Duplicate value"},
+	}, {
+		name:     "dup_ipFamily_dualstack",
+		families: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol, api.IPv6Protocol)),
+		expect: []string{"Duplicate value"},
+	}, {
+		name:     "bad_IP",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIPs("garbage")),
+		expect: []string{"must be a valid IP"},
+	}, {
+		name:     "IP_wrong_family",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIPs("2000::1")),
+		expect: []string{"not configured on this cluster"},
+	}, {
+		name:     "IP_doesnt_match_family",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetIPFamilies(api.IPv4Protocol),
+			svctest.SetClusterIPs("2000::1")),
+		expect: []string{"expected an IPv4 value as indicated"},
+	}, {
+		name:     "too_many_IPs_singlestack",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIPs("10.0.0.1", "10.0.0.2")),
+		expect: []string{"no more than one IP for each IP family"},
+	}, {
+		name:     "too_many_IPs_dualstack",
+		families: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIPs("10.0.0.1", "2000::1", "10.0.0.2")),
+		expect: []string{"only hold up to 2 values"},
+	}, {
+		name:     "dup_IPs",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIPs("10.0.0.1", "10.0.0.1")),
+		expect: []string{"no more than one IP for each IP family"},
+	}, {
+		name:     "empty_IP",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIPs("")),
+		expect: []string{"must be empty when", "must be a valid IP"},
+	}, {
+		name:     "None_IP_1",
+		families: []api.IPFamily{api.IPv4Protocol},
+		svc: svctest.MakeService("foo",
+			svctest.SetClusterIPs("10.0.0.1", "None")),
+		expect: []string{"must be a valid IP"},
+	}}
+
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			storage, _, server := newStorage(t, tc.families)
+			defer server.Terminate(t)
+			defer storage.Store.DestroyFunc()
+
+			ctx := genericapirequest.NewDefaultContext()
+			_, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if err == nil {
+				t.Fatalf("unexpected success creating service")
+			}
+			t.Logf("INFO: errstr:\n  %v", err)
+			for _, s := range tc.expect {
+				if !strings.Contains(err.Error(), s) {
+					t.Errorf("expected to find %q in the error:\n  %s", s, err.Error())
+				}
+			}
+		})
+	}
+}
 func TestCreateDeleteReuse(t *testing.T) {
 	testCases := []struct {
 		name string

From ca8cfdcae906cba2107cb5558bcad43ca1d3ac67 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 13 Aug 2021 11:13:06 -0700
Subject: [PATCH 42/79] Svc REST: Fix single<->dual-stack updates

This removes the old rest_tests and adds significantly more coverage.
Maybe too much.  The v4,v6 and v6,v4 tables are identical but for the
order of families.

This exposed that `trimFieldsForDualStackDowngrade` is called too late
to do anything (since we don't run strategy twice any more).  I moved
similar logic to `PatchAllocatedValues` but I hit on some unclarity.

Specifically, consider a PATCH operation.

Assume I have a valid dual-stack service (with 2 IPs, 2 families, and
policy either require or prefer). What fields can I patch, on their own,
to trigger a downgrade to single-stack?

I think patching policy to "single" is pretty clear intent.

But what if I leave policy and only patch `ipFamilies` down to a single
value (without violating the "can't change first family" rule)?

Or what if I patch `clusterIPs` down to a single IP value?

After much discussion, we decided to make a small API change (OK since
we are beta).  When you want a dual-stack Service you MUST specify the
`ipFamilyPolicy`.  Now we can infer less and avoid ambiguity.
---
 pkg/registry/core/service/storage/rest.go     |   20 +-
 .../core/service/storage/rest_test.go         |  473 ---
 .../core/service/storage/storage_test.go      | 3191 ++++++++++++++++-
 pkg/registry/core/service/strategy.go         |   33 +-
 pkg/registry/core/service/strategy_test.go    |  150 +-
 5 files changed, 3211 insertions(+), 656 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 6e8ea1d3f33..4ccbb5415d3 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -721,29 +721,14 @@ func isMatchingPreferDualStackClusterIPFields(oldService, service *api.Service)
 		return false
 	}
 
-	// compare ClusterIPs lengths.
-	// due to validation.
-	if len(service.Spec.ClusterIPs) != len(oldService.Spec.ClusterIPs) {
+	if !sameClusterIPs(oldService, service) {
 		return false
 	}
 
-	for i, ip := range service.Spec.ClusterIPs {
-		if oldService.Spec.ClusterIPs[i] != ip {
-			return false
-		}
-	}
-
-	// compare IPFamilies
-	if len(service.Spec.IPFamilies) != len(oldService.Spec.IPFamilies) {
+	if !sameIPFamilies(oldService, service) {
 		return false
 	}
 
-	for i, family := range service.Spec.IPFamilies {
-		if oldService.Spec.IPFamilies[i] != family {
-			return false
-		}
-	}
-
 	// they match on
 	// Policy: preferDualStack
 	// ClusterIPs
@@ -865,6 +850,7 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 			// Update: As long as ClusterIPs and IPFamilies have not changed,
 			// setting policy to single-stack is clear intent.
 			if *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicySingleStack {
+				// ClusterIPs[0] is immutable, so it is safe to keep.
 				if sameClusterIPs(oldService, service) && len(service.Spec.ClusterIPs) > 1 {
 					service.Spec.ClusterIPs = service.Spec.ClusterIPs[0:1]
 				}
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index c79d7532aad..f42093322ef 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -26,7 +26,6 @@ import (
 	"sort"
 	"testing"
 
-	"k8s.io/apimachinery/pkg/api/errors"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -40,11 +39,8 @@ import (
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/util/dryrun"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
-	"k8s.io/kubernetes/pkg/features"
 	endpointstore "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
 	podstore "k8s.io/kubernetes/pkg/registry/core/pod/storage"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
@@ -676,56 +672,6 @@ func TestServiceRegistryList(t *testing.T) {
 	}
 }
 
-func TestServiceRegistryIPUpdate(t *testing.T) {
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	svc := svctest.MakeService("foo")
-	ctx := genericapirequest.NewDefaultContext()
-	createdSvc, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	createdService := createdSvc.(*api.Service)
-	if createdService.Spec.Ports[0].Port != svc.Spec.Ports[0].Port {
-		t.Errorf("Expected port %d, but got %v", svc.Spec.Ports[0].Port, createdService.Spec.Ports[0].Port)
-	}
-	if !makeIPNet(t).Contains(netutils.ParseIPSloppy(createdService.Spec.ClusterIPs[0])) {
-		t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIPs[0])
-	}
-
-	update := createdService.DeepCopy()
-	update.Spec.Ports[0].Port = 6503
-
-	updatedSvc, _, errUpdate := storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-	if errUpdate != nil {
-		t.Fatalf("unexpected error during update %v", errUpdate)
-	}
-	updatedService := updatedSvc.(*api.Service)
-	if updatedService.Spec.Ports[0].Port != 6503 {
-		t.Errorf("Expected port 6503, but got %v", updatedService.Spec.Ports[0].Port)
-	}
-
-	testIPs := []string{"1.2.3.93", "1.2.3.94", "1.2.3.95", "1.2.3.96"}
-	testIP := ""
-	for _, ip := range testIPs {
-		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[storage.alloc.defaultServiceIPFamily].(*ipallocator.Range), ip) {
-			testIP = ip
-			break
-		}
-	}
-
-	update = updatedService.DeepCopy()
-	update.Spec.Ports[0].Port = 6503
-	update.Spec.ClusterIP = testIP
-	update.Spec.ClusterIPs[0] = testIP
-
-	_, _, err = storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-	if err == nil || !errors.IsInvalid(err) {
-		t.Errorf("Unexpected error type: %v", err)
-	}
-}
-
 // Validate the internalTrafficPolicy field when set to "Cluster" then updated to "Local"
 func TestServiceRegistryInternalTrafficPolicyClusterThenLocal(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
@@ -789,422 +735,3 @@ func TestServiceRegistryInternalTrafficPolicyLocalThenCluster(t *testing.T) {
 		t.Errorf("Expected internalTrafficPolicy to be Cluster, got: %s", *updatedService.Spec.InternalTrafficPolicy)
 	}
 }
-
-func TestServiceUpgrade(t *testing.T) {
-	requireDualStack := api.IPFamilyPolicyRequireDualStack
-
-	ctx := genericapirequest.NewDefaultContext()
-	testCases := []struct {
-		name                     string
-		updateFunc               func(svc *api.Service)
-		enableDualStackAllocator bool
-		enableDualStackGate      bool
-		allocateIPsBeforeUpdate  map[api.IPFamily]string
-		expectUpgradeError       bool
-		svc                      *api.Service
-	}{{
-		name:                     "normal, no upgrade needed",
-		enableDualStackAllocator: false,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  nil,
-		expectUpgradeError:       false,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.Selector = map[string]string{"bar": "baz2"}
-		},
-
-		svc: svctest.MakeService("foo"),
-	}, {
-		name:                     "error, no upgrade (has single allocator)",
-		enableDualStackAllocator: false,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  nil,
-		expectUpgradeError:       true,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requireDualStack
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
-		}),
-	}, {
-		name:                     "upgrade to v4,6",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  nil,
-		expectUpgradeError:       false,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requireDualStack
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
-		}),
-	}, {
-		name:                     "upgrade to v4,6 (specific ip)",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  nil,
-		expectUpgradeError:       false,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requireDualStack
-			s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "2000:0:0:0:0:0:0:1")
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
-		}),
-	}, {
-		name:                     "upgrade to v4,6 (specific ip) - fail, ip is not available",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  map[api.IPFamily]string{api.IPv6Protocol: "2000:0:0:0:0:0:0:1"},
-		expectUpgradeError:       true,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requireDualStack
-			s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "2000:0:0:0:0:0:0:1")
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
-		}),
-	}, {
-		name:                     "upgrade to v6,4",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  nil,
-		expectUpgradeError:       false,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requireDualStack
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol}
-		}),
-	}, {
-		name:                     "upgrade to v6,4 (specific ip)",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  nil,
-		expectUpgradeError:       false,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requireDualStack
-			s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "1.2.3.4")
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol}
-		}),
-	}, {
-		name:                     "upgrade to v6,4 (specific ip) - fail ip is already allocated",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		allocateIPsBeforeUpdate:  map[api.IPFamily]string{api.IPv4Protocol: "1.2.3.4"},
-		expectUpgradeError:       true,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requireDualStack
-			s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "1.2.3.4")
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol}
-		}),
-	}}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			families := []api.IPFamily{api.IPv4Protocol}
-			if testCase.enableDualStackAllocator {
-				families = append(families, api.IPv6Protocol)
-			}
-			storage, server := NewTestREST(t, families)
-			defer server.Terminate(t)
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, testCase.enableDualStackGate)()
-
-			obj, err := storage.Create(ctx, testCase.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("error is unexpected: %v", err)
-			}
-
-			createdSvc := obj.(*api.Service)
-			// allocated IP
-			for family, ip := range testCase.allocateIPsBeforeUpdate {
-				alloc := storage.alloc.serviceIPAllocatorsByFamily[family]
-				if err := alloc.Allocate(netutils.ParseIPSloppy(ip)); err != nil {
-					t.Fatalf("test is incorrect, unable to preallocate ip:%v", ip)
-				}
-			}
-			// run the modifier
-			testCase.updateFunc(createdSvc)
-
-			// run the update
-			updated, _, err := storage.Update(ctx,
-				createdSvc.Name,
-				rest.DefaultUpdatedObjectInfo(createdSvc),
-				rest.ValidateAllObjectFunc,
-				rest.ValidateAllObjectUpdateFunc,
-				false,
-				&metav1.UpdateOptions{})
-
-			if err != nil && !testCase.expectUpgradeError {
-				t.Fatalf("an error was not expected during upgrade %v", err)
-			}
-
-			if err == nil && testCase.expectUpgradeError {
-				t.Fatalf("error was expected during upgrade")
-			}
-
-			if err != nil {
-				return
-			}
-
-			updatedSvc := updated.(*api.Service)
-			isValidClusterIPFields(t, storage, updatedSvc, updatedSvc)
-
-			shouldUpgrade := len(createdSvc.Spec.IPFamilies) == 2 && *(createdSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.alloc.serviceIPAllocatorsByFamily) == 2
-			if shouldUpgrade && len(updatedSvc.Spec.ClusterIPs) < 2 {
-				t.Fatalf("Service should have been upgraded %+v", createdSvc)
-			}
-
-			if !shouldUpgrade && len(updatedSvc.Spec.ClusterIPs) > 1 {
-				t.Fatalf("Service should not have been upgraded %+v", createdSvc)
-			}
-
-			// make sure that ips were allocated, correctly
-			for i, family := range updatedSvc.Spec.IPFamilies {
-				ip := updatedSvc.Spec.ClusterIPs[i]
-				allocator := storage.alloc.serviceIPAllocatorsByFamily[family]
-				if !ipIsAllocated(t, allocator, ip) {
-					t.Fatalf("expected ip:%v to be allocated by %v allocator. it was not", ip, family)
-				}
-			}
-		})
-	}
-}
-
-func TestServiceDowngrade(t *testing.T) {
-	requiredDualStack := api.IPFamilyPolicyRequireDualStack
-	singleStack := api.IPFamilyPolicySingleStack
-	ctx := genericapirequest.NewDefaultContext()
-	testCases := []struct {
-		name                     string
-		updateFunc               func(svc *api.Service)
-		enableDualStackAllocator bool
-		enableDualStackGate      bool
-		expectDowngradeError     bool
-		svc                      *api.Service
-	}{{
-		name:                     "normal, no downgrade needed. single stack => single stack",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		expectDowngradeError:     false,
-
-		updateFunc: func(s *api.Service) { s.Spec.Selector = map[string]string{"bar": "baz2"} },
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requiredDualStack
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol}
-		}),
-	}, {
-		name:                     "normal, no downgrade needed. dual stack => dual stack",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		expectDowngradeError:     false,
-
-		updateFunc: func(s *api.Service) { s.Spec.Selector = map[string]string{"bar": "baz2"} },
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requiredDualStack
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-		}),
-	}, {
-		name:                     "normal, downgrade v4,v6 => v4",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		expectDowngradeError:     false,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &singleStack
-			s.Spec.ClusterIPs = s.Spec.ClusterIPs[0:1]
-			s.Spec.IPFamilies = s.Spec.IPFamilies[0:1]
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requiredDualStack
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-		}),
-	}, {
-		name:                     "normal, downgrade v6,v4 => v6",
-		enableDualStackAllocator: true,
-		enableDualStackGate:      true,
-		expectDowngradeError:     false,
-
-		updateFunc: func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &singleStack
-			s.Spec.ClusterIPs = s.Spec.ClusterIPs[0:1]
-			s.Spec.IPFamilies = s.Spec.IPFamilies[0:1]
-		},
-
-		svc: svctest.MakeService("foo", func(s *api.Service) {
-			s.Spec.IPFamilyPolicy = &requiredDualStack
-			s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}
-		}),
-	}}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
-			defer server.Terminate(t)
-			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, testCase.enableDualStackGate)()
-
-			obj, err := storage.Create(ctx, testCase.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("error is unexpected: %v", err)
-			}
-
-			createdSvc := obj.(*api.Service)
-			copySvc := createdSvc.DeepCopy()
-
-			// run the modifier
-			testCase.updateFunc(createdSvc)
-
-			// run the update
-			updated, _, err := storage.Update(ctx,
-				createdSvc.Name,
-				rest.DefaultUpdatedObjectInfo(createdSvc),
-				rest.ValidateAllObjectFunc,
-				rest.ValidateAllObjectUpdateFunc,
-				false,
-				&metav1.UpdateOptions{})
-
-			if err != nil && !testCase.expectDowngradeError {
-				t.Fatalf("an error was not expected during upgrade %v", err)
-			}
-
-			if err == nil && testCase.expectDowngradeError {
-				t.Fatalf("error was expected during upgrade")
-			}
-
-			if err != nil {
-				return
-			}
-
-			updatedSvc := updated.(*api.Service)
-			isValidClusterIPFields(t, storage, createdSvc, updatedSvc)
-
-			shouldDowngrade := len(copySvc.Spec.ClusterIPs) == 2 && *(createdSvc.Spec.IPFamilyPolicy) == api.IPFamilyPolicySingleStack
-
-			if shouldDowngrade && len(updatedSvc.Spec.ClusterIPs) > 1 {
-				t.Fatalf("Service should have been downgraded %+v", createdSvc)
-			}
-
-			if !shouldDowngrade && len(updatedSvc.Spec.ClusterIPs) < 2 {
-				t.Fatalf("Service should not have been downgraded %+v", createdSvc)
-			}
-
-			if shouldDowngrade {
-				releasedIP := copySvc.Spec.ClusterIPs[1]
-				releasedIPFamily := copySvc.Spec.IPFamilies[1]
-				allocator := storage.alloc.serviceIPAllocatorsByFamily[releasedIPFamily]
-
-				if ipIsAllocated(t, allocator, releasedIP) {
-					t.Fatalf("expected ip:%v to be released by %v allocator. it was not", releasedIP, releasedIPFamily)
-				}
-			}
-		})
-	}
-}
-
-// validates that the service created, updated by REST
-// has correct ClusterIPs related fields
-func isValidClusterIPFields(t *testing.T, storage *REST, pre *api.Service, post *api.Service) {
-	t.Helper()
-
-	// valid for gate off/on scenarios
-	// ClusterIP
-	if len(post.Spec.ClusterIP) == 0 {
-		t.Fatalf("service must have clusterIP : %+v", post)
-	}
-	// cluster IPs
-	if len(post.Spec.ClusterIPs) == 0 {
-		t.Fatalf("new service must have at least one IP: %+v", post)
-	}
-
-	if post.Spec.ClusterIP != post.Spec.ClusterIPs[0] {
-		t.Fatalf("clusterIP does not match ClusterIPs[0]: %+v", post)
-	}
-
-	// if feature gate is not enabled then we need to ignore need fields
-	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		if post.Spec.IPFamilyPolicy != nil {
-			t.Fatalf("service must be set to  nil for IPFamilyPolicy: %+v", post)
-		}
-
-		if len(post.Spec.IPFamilies) != 0 {
-			t.Fatalf("service must be set to nil for IPFamilies: %+v", post)
-		}
-
-		return
-	}
-
-	// for gate on scenarios
-	// prefer dual stack field
-	if post.Spec.IPFamilyPolicy == nil {
-		t.Fatalf("service must not have nil for IPFamilyPolicy: %+v", post)
-	}
-
-	if pre.Spec.IPFamilyPolicy != nil && *(pre.Spec.IPFamilyPolicy) != *(post.Spec.IPFamilyPolicy) {
-		t.Fatalf("new service must not change PreferDualStack if it was set by user pre: %v post: %v", *(pre.Spec.IPFamilyPolicy), *(post.Spec.IPFamilyPolicy))
-	}
-
-	if pre.Spec.IPFamilyPolicy == nil && *(post.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack {
-		t.Fatalf("new services with prefer dual stack nil must be set to false (prefer dual stack) %+v", post)
-	}
-
-	// external name or headless services offer no more ClusterIPs field validation
-	if post.Spec.ClusterIPs[0] == api.ClusterIPNone {
-		return
-	}
-
-	// len of ClusteIPs can not be more than Families
-	// and for providedIPs it needs to match
-
-	// if families are provided then it shouldn't be changed
-	// this applies on first entry on
-	if len(pre.Spec.IPFamilies) > 0 {
-		if len(post.Spec.IPFamilies) == 0 {
-			t.Fatalf("allocator shouldn't remove ipfamilies[0] pre:%+v, post:%+v", pre.Spec.IPFamilies, post.Spec.IPFamilies)
-		}
-
-		if pre.Spec.IPFamilies[0] != post.Spec.IPFamilies[0] {
-			t.Fatalf("allocator shouldn't change post.Spec.IPFamilies[0] pre:%+v post:%+v", pre.Spec.IPFamilies, post.Spec.IPFamilies)
-		}
-	}
-	// if two families are assigned, then they must be dual stack
-	if len(post.Spec.IPFamilies) == 2 {
-		if post.Spec.IPFamilies[0] == post.Spec.IPFamilies[1] {
-			t.Fatalf("allocator assigned two of the same family %+v", post)
-		}
-	}
-	// ips must match families
-	for i, ip := range post.Spec.ClusterIPs {
-		isIPv6 := netutils.IsIPv6String(ip)
-		if isIPv6 && post.Spec.IPFamilies[i] != api.IPv6Protocol {
-			t.Fatalf("ips does not match assigned families %+v %+v", post.Spec.ClusterIPs, post.Spec.IPFamilies)
-		}
-	}
-}
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 16e9d6593ec..fd26a8408a3 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5975,6 +5975,3145 @@ func TestUpdateDryRun(t *testing.T) {
 	}
 }
 
+// Proves that updates from single-stack to dual-stack work.
+func TestUpdateSingleToDualStack(t *testing.T) {
+	prove := func(proofs ...svcTestProof) []svcTestProof {
+		return proofs
+	}
+	proveNumFamilies := func(n int) svcTestProof {
+		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+			t.Helper()
+			if got := len(after.Spec.IPFamilies); got != n {
+				t.Errorf("wrong number of ipFamilies: expected %d, got %d", n, got)
+			}
+		}
+	}
+
+	// Single-stack cases as control.
+	testCasesV4 := []cudTestCase{{
+		name: "single-single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetSelector(map[string]string{"k2": "v2"})),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "single-dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "single-dual_policy",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+			expectError: true,
+		},
+	}, {
+		name: "single-dual_families",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectError: true,
+		},
+	}}
+
+	t.Run("singlestack:v4", func(t *testing.T) {
+		helpTestCreateUpdateDeleteWithFamilies(t, testCasesV4, []api.IPFamily{api.IPv4Protocol})
+	})
+
+	// Dual-stack v4,v6 cases: Covers the full matrix of:
+	//    policy={nil, single, prefer, require}
+	//    families={nil, single, dual}
+	//    ips={nil, single, dual}
+	testCasesV4V6 := []cudTestCase{{
+		name: "policy:nil_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"})),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "single-dual_wrong_order_families",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "single-dual_wrong_order_ips",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "single-dual_ip_in_use",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		beforeUpdate: func(t *testing.T, storage *GenericREST) {
+			alloc := storage.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol]
+			ip := "2000::1"
+			if err := alloc.Allocate(netutils.ParseIPSloppy(ip)); err != nil {
+				t.Fatalf("test is incorrect, unable to preallocate IP %q: %v", ip, err)
+			}
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}}
+
+	t.Run("dualstack:v4v6", func(t *testing.T) {
+		helpTestCreateUpdateDeleteWithFamilies(t, testCasesV4V6, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	})
+
+	// Dual-stack v6,v4 cases: Covers the full matrix of:
+	//    policy={nil, single, prefer, require}
+	//    families={nil, single, dual}
+	//    ips={nil, single, dual}
+	testCasesV6V4 := []cudTestCase{{
+		name: "policy:nil_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"})),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:single_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "single-dual_wrong_order_families",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "single-dual_wrong_order_ips",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "single-dual_ip_in_use",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+		beforeUpdate: func(t *testing.T, storage *GenericREST) {
+			alloc := storage.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol]
+			ip := "10.0.0.1"
+			if err := alloc.Allocate(netutils.ParseIPSloppy(ip)); err != nil {
+				t.Fatalf("test is incorrect, unable to preallocate IP %q: %v", ip, err)
+			}
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}}
+
+	t.Run("dualstack:v6v4", func(t *testing.T) {
+		helpTestCreateUpdateDeleteWithFamilies(t, testCasesV6V4, []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol})
+	})
+
+	// Headless cases: Covers the full matrix of:
+	//    policy={nil, single, prefer, require}
+	//    families={nil, single, dual}
+	testCasesHeadless := []cudTestCase{{
+		name: "policy:nil_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"})),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies("IPv4")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:single_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies("IPv4")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies("IPv4")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies("IPv4")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}}
+
+	t.Run("headless", func(t *testing.T) {
+		helpTestCreateUpdateDeleteWithFamilies(t, testCasesHeadless, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	})
+}
+
+// Proves that updates from dual-stack to single-stack work.
+func TestUpdateDualToSingleStack(t *testing.T) {
+	prove := func(proofs ...svcTestProof) []svcTestProof {
+		return proofs
+	}
+	proveNumFamilies := func(n int) svcTestProof {
+		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+			t.Helper()
+			if got := len(after.Spec.IPFamilies); got != n {
+				t.Errorf("wrong number of ipFamilies: expected %d, got %d", n, got)
+			}
+		}
+	}
+
+	// Dual-stack v4,v6 cases: Covers the full matrix of:
+	//    policy={nil, single, prefer, require}
+	//    families={nil, single, dual}
+	//    ips={nil, single, dual}
+	testCasesV4V6 := []cudTestCase{{
+		name: "policy:nil_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"})),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "dual-single_wrong_order_families",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "dual-single_wrong_order_ips",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}}
+
+	t.Run("dualstack:v4v6", func(t *testing.T) {
+		helpTestCreateUpdateDeleteWithFamilies(t, testCasesV4V6, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	})
+
+	// Dual-stack v6,v4 cases: Covers the full matrix of:
+	//    policy={nil, single, prefer, require}
+	//    families={nil, single, dual}
+	//    ips={nil, single, dual}
+	testCasesV6V4 := []cudTestCase{{
+		name: "policy:nil_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"})),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs:     true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:nil_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:single_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:single_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:dual_ips:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:dual_ips:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:dual_ips:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "dual-single_wrong_order_families",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies(api.IPv4Protocol)),
+			expectError: true,
+		},
+	}, {
+		name: "dual-single_wrong_order_ips",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
+			expectClusterIPs: true,
+			prove:            prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}}
+
+	t.Run("dualstack:v6v4", func(t *testing.T) {
+		helpTestCreateUpdateDeleteWithFamilies(t, testCasesV6V4, []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol})
+	})
+
+	// Headless cases: Covers the full matrix of:
+	//    policy={nil, single, prefer, require}
+	//    families={nil, single, dual}
+	testCasesHeadless := []cudTestCase{{
+		name: "policy:nil_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"})),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:nil_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies("IPv4")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:nil_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:single_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies("IPv4")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:single_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectHeadless:       true,
+			expectStackDowngrade: true,
+			prove:                prove(proveNumFamilies(1)),
+		},
+	}, {
+		name: "policy:prefer_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:prefer_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies("IPv4")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:prefer_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:nil",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}, {
+		name: "policy:require_families:single",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies("IPv4")),
+			expectError: true,
+		},
+	}, {
+		name: "policy:require_families:dual",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetClusterIPs(api.ClusterIPNone)),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetSelector(map[string]string{"k2": "v2"}),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
+				svctest.SetIPFamilies("IPv4", "IPv6")),
+			expectHeadless: true,
+			prove:          prove(proveNumFamilies(2)),
+		},
+	}}
+
+	t.Run("headless", func(t *testing.T) {
+		helpTestCreateUpdateDeleteWithFamilies(t, testCasesHeadless, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+	})
+}
+
 type svcTestCase struct {
 	svc         *api.Service
 	expectError bool
@@ -5983,6 +9122,7 @@ type svcTestCase struct {
 	// vector for test bugs and more importantly it makes the test cases less
 	// self-documenting.
 	expectClusterIPs          bool
+	expectStackDowngrade      bool
 	expectHeadless            bool
 	expectNodePorts           bool
 	expectHealthCheckNodePort bool
@@ -6009,10 +9149,8 @@ func callName(before, after *api.Service) string {
 func proveClusterIPsAllocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
 	t.Helper()
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
-			t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
-		}
+	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
+		t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
 	}
 
 	clips := []string{}
@@ -6039,20 +9177,46 @@ func proveClusterIPsAllocated(t *testing.T, storage *GenericREST, before, after
 		}
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		if after.Spec.IPFamilyPolicy == nil {
+			t.Errorf("%s: expected ipFamilyPolicy to be set", callName(before, after))
+		} else {
+			pol := *after.Spec.IPFamilyPolicy
+			fams := len(after.Spec.IPFamilies)
+			clus := 1
+			if storage.secondaryIPFamily != "" {
+				clus = 2
+			}
+			if pol == api.IPFamilyPolicySingleStack && fams != 1 {
+				t.Errorf("%s: expected 1 ipFamily, got %d", callName(before, after), fams)
+			} else if pol == api.IPFamilyPolicyRequireDualStack && fams != 2 {
+				t.Errorf("%s: expected 2 ipFamilies, got %d", callName(before, after), fams)
+			} else if pol == api.IPFamilyPolicyPreferDualStack && fams != clus {
+				t.Errorf("%s: expected %d ipFamilies, got %d", callName(before, after), clus, fams)
+			}
+		}
+	}
+
 	if before != nil {
 		if before.Spec.ClusterIP != "" {
 			if want, got := before.Spec.ClusterIP, after.Spec.ClusterIP; want != got {
 				t.Errorf("%s: wrong clusterIP: wanted %q, got %q", callName(before, after), want, got)
 			}
 		}
-		for i := range before.Spec.ClusterIPs {
+		min := func(x, y int) int {
+			if x < y {
+				return x
+			}
+			return y
+		}
+		for i := 0; i < min(len(before.Spec.ClusterIPs), len(after.Spec.ClusterIPs)); i++ {
 			if want, got := before.Spec.ClusterIPs[i], after.Spec.ClusterIPs[i]; want != got {
 				t.Errorf("%s: wrong clusterIPs[%d]: wanted %q, got %q", callName(before, after), i, want, got)
 			}
 		}
-		for i := range before.Spec.IPFamilies {
+		for i := 0; i < min(len(before.Spec.IPFamilies), len(after.Spec.IPFamilies)); i++ {
 			if want, got := before.Spec.IPFamilies[i], after.Spec.IPFamilies[i]; want != got {
-				t.Errorf("%s: wrong IPFamilies[%d]: wanted %q, got %q", callName(before, after), i, want, got)
+				t.Errorf("%s: wrong ipFamilies[%d]: wanted %q, got %q", callName(before, after), i, want, got)
 			}
 		}
 	}
@@ -6266,17 +9430,22 @@ func verifyEquiv(t testingTInterface, call string, tc *svcTestCase, got *api.Ser
 			want.Spec.ClusterIP = got.Spec.ClusterIP
 		}
 		if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-			if len(got.Spec.ClusterIPs) > len(want.Spec.ClusterIPs) {
-				want.Spec.ClusterIPs = append(want.Spec.ClusterIPs, got.Spec.ClusterIPs[len(want.Spec.ClusterIPs):]...)
-			}
 			if want.Spec.IPFamilyPolicy == nil {
 				want.Spec.IPFamilyPolicy = got.Spec.IPFamilyPolicy
 			}
-			if len(got.Spec.IPFamilies) > len(want.Spec.IPFamilies) {
+			if tc.expectStackDowngrade && len(want.Spec.ClusterIPs) > len(got.Spec.ClusterIPs) {
+				want.Spec.ClusterIPs = want.Spec.ClusterIPs[0:1]
+			} else if len(got.Spec.ClusterIPs) > len(want.Spec.ClusterIPs) {
+				want.Spec.ClusterIPs = append(want.Spec.ClusterIPs, got.Spec.ClusterIPs[len(want.Spec.ClusterIPs):]...)
+			}
+			if tc.expectStackDowngrade && len(want.Spec.IPFamilies) > len(got.Spec.ClusterIPs) {
+				want.Spec.IPFamilies = want.Spec.IPFamilies[0:1]
+			} else if len(got.Spec.IPFamilies) > len(want.Spec.IPFamilies) {
 				want.Spec.IPFamilies = append(want.Spec.IPFamilies, got.Spec.IPFamilies[len(want.Spec.IPFamilies):]...)
 			}
 		}
 	}
+
 	if tc.expectNodePorts {
 		for i := range want.Spec.Ports {
 			p := &want.Spec.Ports[i]
diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go
index 0a34b59876c..346cf2db6eb 100644
--- a/pkg/registry/core/service/strategy.go
+++ b/pkg/registry/core/service/strategy.go
@@ -124,7 +124,6 @@ func (strategy svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runti
 	NormalizeClusterIPs(oldService, newService)
 	dropServiceDisabledFields(newService, oldService)
 	dropTypeDependentFields(newService, oldService)
-	trimFieldsForDualStackDowngrade(newService, oldService)
 }
 
 // Validate validates a new service.
@@ -314,7 +313,7 @@ func PatchAllocatedValues(newSvc, oldSvc *api.Service) {
 		if newSvc.Spec.ClusterIP == "" {
 			newSvc.Spec.ClusterIP = oldSvc.Spec.ClusterIP
 		}
-		if len(newSvc.Spec.ClusterIPs) == 0 {
+		if len(newSvc.Spec.ClusterIPs) == 0 && len(oldSvc.Spec.ClusterIPs) > 0 {
 			newSvc.Spec.ClusterIPs = oldSvc.Spec.ClusterIPs
 		}
 	}
@@ -620,33 +619,3 @@ func needsExternalTrafficPolicy(svc *api.Service) bool {
 func sameExternalTrafficPolicy(oldSvc, newSvc *api.Service) bool {
 	return oldSvc.Spec.ExternalTrafficPolicy == newSvc.Spec.ExternalTrafficPolicy
 }
-
-// this func allows user to downgrade a service by just changing
-// IPFamilyPolicy to SingleStack
-func trimFieldsForDualStackDowngrade(newService, oldService *api.Service) {
-	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return
-	}
-
-	// not an update
-	if oldService == nil {
-		return
-	}
-
-	oldIsDualStack := oldService.Spec.IPFamilyPolicy != nil &&
-		(*oldService.Spec.IPFamilyPolicy == api.IPFamilyPolicyRequireDualStack ||
-			*oldService.Spec.IPFamilyPolicy == api.IPFamilyPolicyPreferDualStack)
-
-	newIsNotDualStack := newService.Spec.IPFamilyPolicy != nil && *newService.Spec.IPFamilyPolicy == api.IPFamilyPolicySingleStack
-
-	// if user want to downgrade then we auto remove secondary ip and family
-	if oldIsDualStack && newIsNotDualStack {
-		if len(newService.Spec.ClusterIPs) > 1 {
-			newService.Spec.ClusterIPs = newService.Spec.ClusterIPs[0:1]
-		}
-
-		if len(newService.Spec.IPFamilies) > 1 {
-			newService.Spec.IPFamilies = newService.Spec.IPFamilies[0:1]
-		}
-	}
-}
diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go
index 5603c90a766..0c9a0f31db2 100644
--- a/pkg/registry/core/service/strategy_test.go
+++ b/pkg/registry/core/service/strategy_test.go
@@ -930,123 +930,15 @@ func TestDropTypeDependentFields(t *testing.T) {
 	}
 }
 
-func TestTrimFieldsForDualStackDowngrade(t *testing.T) {
-	singleStack := api.IPFamilyPolicySingleStack
-	preferDualStack := api.IPFamilyPolicyPreferDualStack
-	requireDualStack := api.IPFamilyPolicyRequireDualStack
-	testCases := []struct {
-		name          string
-		oldPolicy     *api.IPFamilyPolicyType
-		oldClusterIPs []string
-		oldFamilies   []api.IPFamily
-
-		newPolicy          *api.IPFamilyPolicyType
-		expectedClusterIPs []string
-		expectedIPFamilies []api.IPFamily
-	}{
-
-		{
-			name:               "no change single to single",
-			oldPolicy:          &singleStack,
-			oldClusterIPs:      []string{"10.10.10.10"},
-			oldFamilies:        []api.IPFamily{api.IPv4Protocol},
-			newPolicy:          &singleStack,
-			expectedClusterIPs: []string{"10.10.10.10"},
-			expectedIPFamilies: []api.IPFamily{api.IPv4Protocol},
-		},
-
-		{
-			name:               "dualstack to dualstack (preferred)",
-			oldPolicy:          &preferDualStack,
-			oldClusterIPs:      []string{"10.10.10.10", "2000::1"},
-			oldFamilies:        []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			newPolicy:          &preferDualStack,
-			expectedClusterIPs: []string{"10.10.10.10", "2000::1"},
-			expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-		},
-
-		{
-			name:               "dualstack to dualstack (required)",
-			oldPolicy:          &requireDualStack,
-			oldClusterIPs:      []string{"10.10.10.10", "2000::1"},
-			oldFamilies:        []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			newPolicy:          &preferDualStack,
-			expectedClusterIPs: []string{"10.10.10.10", "2000::1"},
-			expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-		},
-
-		{
-			name:               "dualstack (preferred) to single",
-			oldPolicy:          &preferDualStack,
-			oldClusterIPs:      []string{"10.10.10.10", "2000::1"},
-			oldFamilies:        []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-			newPolicy:          &singleStack,
-			expectedClusterIPs: []string{"10.10.10.10"},
-			expectedIPFamilies: []api.IPFamily{api.IPv4Protocol},
-		},
-
-		{
-			name:               "dualstack (require) to single",
-			oldPolicy:          &requireDualStack,
-			oldClusterIPs:      []string{"2000::1", "10.10.10.10"},
-			oldFamilies:        []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-			newPolicy:          &singleStack,
-			expectedClusterIPs: []string{"2000::1"},
-			expectedIPFamilies: []api.IPFamily{api.IPv6Protocol},
-		},
-	}
-	// only when gate is on
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			oldService := &api.Service{
-				Spec: api.ServiceSpec{
-					IPFamilyPolicy: tc.oldPolicy,
-					ClusterIPs:     tc.oldClusterIPs,
-					IPFamilies:     tc.oldFamilies,
-				},
-			}
-
-			newService := oldService.DeepCopy()
-			newService.Spec.IPFamilyPolicy = tc.newPolicy
-
-			trimFieldsForDualStackDowngrade(newService, oldService)
-
-			if len(newService.Spec.ClusterIPs) != len(tc.expectedClusterIPs) {
-				t.Fatalf("unexpected clusterIPs. expected %v and got %v", tc.expectedClusterIPs, newService.Spec.ClusterIPs)
-			}
-
-			// compare clusterIPS
-			for i, expectedIP := range tc.expectedClusterIPs {
-				if expectedIP != newService.Spec.ClusterIPs[i] {
-					t.Fatalf("unexpected clusterIPs. expected %v and got %v", tc.expectedClusterIPs, newService.Spec.ClusterIPs)
-				}
-			}
-
-			// families
-			if len(newService.Spec.IPFamilies) != len(tc.expectedIPFamilies) {
-				t.Fatalf("unexpected ipfamilies. expected %v and got %v", tc.expectedIPFamilies, newService.Spec.IPFamilies)
-			}
-
-			// compare clusterIPS
-			for i, expectedIPFamily := range tc.expectedIPFamilies {
-				if expectedIPFamily != newService.Spec.IPFamilies[i] {
-					t.Fatalf("unexpected ipfamilies. expected %v and got %v", tc.expectedIPFamilies, newService.Spec.IPFamilies)
-				}
-			}
-
-		})
-	}
-}
-
 func TestPatchAllocatedValues(t *testing.T) {
 	testCases := []struct {
-		name                 string
-		before               *api.Service
-		update               *api.Service
-		expectSameClusterIPs bool
-		expectSameNodePort   bool
-		expectSameHCNP       bool
+		name                    string
+		before                  *api.Service
+		update                  *api.Service
+		expectSameClusterIPs    bool
+		expectReducedClusterIPs bool
+		expectSameNodePort      bool
+		expectSameHCNP          bool
 	}{{
 		name: "all_patched",
 		before: svctest.MakeService("foo",
@@ -1120,16 +1012,28 @@ func TestPatchAllocatedValues(t *testing.T) {
 			update := tc.update.DeepCopy()
 			PatchAllocatedValues(update, tc.before)
 
-			if b, u := tc.before.Spec.ClusterIP, update.Spec.ClusterIP; tc.expectSameClusterIPs && b != u {
-				t.Errorf("expected clusterIP to be patched: %q != %q", b, u)
-			} else if !tc.expectSameClusterIPs && b == u {
-				t.Errorf("expected clusterIP to not be patched: %q == %q", b, u)
+			beforeIP := tc.before.Spec.ClusterIP
+			updateIP := update.Spec.ClusterIP
+			if tc.expectSameClusterIPs || tc.expectReducedClusterIPs {
+				if beforeIP != updateIP {
+					t.Errorf("expected clusterIP to be patched: %q != %q", beforeIP, updateIP)
+				}
+			} else if beforeIP == updateIP {
+				t.Errorf("expected clusterIP to not be patched: %q == %q", beforeIP, updateIP)
 			}
 
-			if b, u := tc.before.Spec.ClusterIPs, update.Spec.ClusterIPs; tc.expectSameClusterIPs && !cmp.Equal(b, u) {
-				t.Errorf("expected clusterIPs to be patched: %q != %q", b, u)
-			} else if !tc.expectSameClusterIPs && cmp.Equal(b, u) {
-				t.Errorf("expected clusterIPs to not be patched: %q == %q", b, u)
+			beforeIPs := tc.before.Spec.ClusterIPs
+			updateIPs := update.Spec.ClusterIPs
+			if tc.expectSameClusterIPs {
+				if !cmp.Equal(beforeIPs, updateIPs) {
+					t.Errorf("expected clusterIPs to be patched: %q != %q", beforeIPs, updateIPs)
+				}
+			} else if tc.expectReducedClusterIPs {
+				if len(updateIPs) != 1 || beforeIPs[0] != updateIPs[0] {
+					t.Errorf("expected clusterIPs to be trim-patched: %q -> %q", beforeIPs, updateIPs)
+				}
+			} else if cmp.Equal(beforeIPs, updateIPs) {
+				t.Errorf("expected clusterIPs to not be patched: %q == %q", beforeIPs, updateIPs)
 			}
 
 			if b, u := tc.before.Spec.Ports[0].NodePort, update.Spec.Ports[0].NodePort; tc.expectSameNodePort && b != u {

From d30ae6a5abf333705bb10519635ffa14ada3c86f Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 18 Aug 2021 17:05:07 -0700
Subject: [PATCH 43/79] Svc REST: Make ipFamilyPolicy authoritative

Previously we would try to infer the `ipFamilyPolicy` from `clusterIPs`
and/or `ipFamilies`.  That is too tricky.  Now you MUST specify
`ipFamilyPolicy` as one of the dual-stack options in order to get a
dual-stack service.
---
 pkg/registry/core/service/storage/rest.go     | 108 ++-
 .../core/service/storage/rest_test.go         |   2 +
 .../core/service/storage/storage_test.go      | 862 ++++++++++++++++--
 test/e2e/network/dual_stack.go                |   4 +-
 4 files changed, 832 insertions(+), 144 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 4ccbb5415d3..c0dbf401f6f 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -824,42 +824,34 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		return nil // nothing more to do.
 	}
 
-	// Update-only prep work.
-	if oldService != nil {
-		np := service.Spec.IPFamilyPolicy
-
-		// If they didn't specify policy, or specified anything but
-		// single-stack AND they reduced these fields, it's an error.  They
-		// need to specify policy.
-		if np == nil || *np != api.IPFamilyPolicySingleStack {
-			el := make(field.ErrorList, 0)
-
-			if reducedClusterIPs(oldService, service) {
-				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
-					"must be 'SingleStack' to release the secondary cluster IP"))
-			}
-			if reducedIPFamilies(oldService, service) {
-				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
-					"must be 'SingleStack' to release the secondary IP family"))
-			}
-
-			if len(el) > 0 {
-				return errors.NewInvalid(api.Kind("Service"), service.Name, el)
-			}
-		} else { // policy must be SingleStack
-			// Update: As long as ClusterIPs and IPFamilies have not changed,
-			// setting policy to single-stack is clear intent.
-			if *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicySingleStack {
-				// ClusterIPs[0] is immutable, so it is safe to keep.
-				if sameClusterIPs(oldService, service) && len(service.Spec.ClusterIPs) > 1 {
-					service.Spec.ClusterIPs = service.Spec.ClusterIPs[0:1]
-				}
-				if sameIPFamilies(oldService, service) && len(service.Spec.IPFamilies) > 1 {
-					service.Spec.IPFamilies = service.Spec.IPFamilies[0:1]
-				}
-			}
+	// If the user didn't specify ipFamilyPolicy, we can infer a default.  We
+	// don't want a static default because we want to make sure that we never
+	// change between single- and dual-stack modes with explicit direction, as
+	// provided by ipFamilyPolicy.  Consider these cases:
+	//   * Create (POST): If they didn't specify a policy we can assume it's
+	//     always SingleStack.
+	//   * Update (PUT): If they didn't specify a policy we need to adopt the
+	//     policy from before.  This is better than always assuming SingleStack
+	//     because a PUT that changes clusterIPs from 2 to 1 value but doesn't
+	//     specify ipFamily would work.
+	//   * Update (PATCH): If they didn't specify a policy it will adopt the
+	//     policy from before.
+	if service.Spec.IPFamilyPolicy == nil {
+		if oldService != nil && oldService.Spec.IPFamilyPolicy != nil {
+			// Update from an object with policy, use the old policy
+			service.Spec.IPFamilyPolicy = oldService.Spec.IPFamilyPolicy
+		} else if service.Spec.ClusterIP == api.ClusterIPNone && len(service.Spec.Selector) == 0 {
+			// Special-case: headless + selectorless defaults to dual.
+			requireDualStack := api.IPFamilyPolicyRequireDualStack
+			service.Spec.IPFamilyPolicy = &requireDualStack
+		} else {
+			// create or update from an object without policy (e.g.
+			// ExternalName) to one that needs policy
+			singleStack := api.IPFamilyPolicySingleStack
+			service.Spec.IPFamilyPolicy = &singleStack
 		}
 	}
+	// Henceforth we can assume ipFamilyPolicy is set.
 
 	// Do some loose pre-validation of the input.  This makes it easier in the
 	// rest of allocation code to not have to consider corner cases.
@@ -872,6 +864,32 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 	//TODO(thockin): Move this logic to validation?
 	el := make(field.ErrorList, 0)
 
+	// Update-only prep work.
+	if oldService != nil {
+		if getIPFamilyPolicy(service) == api.IPFamilyPolicySingleStack {
+			// As long as ClusterIPs and IPFamilies have not changed, setting
+			// the policy to single-stack is clear intent.
+			// ClusterIPs[0] is immutable, so it is safe to keep.
+			if sameClusterIPs(oldService, service) && len(service.Spec.ClusterIPs) > 1 {
+				service.Spec.ClusterIPs = service.Spec.ClusterIPs[0:1]
+			}
+			if sameIPFamilies(oldService, service) && len(service.Spec.IPFamilies) > 1 {
+				service.Spec.IPFamilies = service.Spec.IPFamilies[0:1]
+			}
+		} else {
+			// If the policy is anything but single-stack AND they reduced these
+			// fields, it's an error.  They need to specify policy.
+			if reducedClusterIPs(oldService, service) {
+				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
+					"must be 'SingleStack' to release the secondary cluster IP"))
+			}
+			if reducedIPFamilies(oldService, service) {
+				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
+					"must be 'SingleStack' to release the secondary IP family"))
+			}
+		}
+	}
+
 	// Make sure ipFamilyPolicy makes sense for the provided ipFamilies and
 	// clusterIPs.  Further checks happen below - after the special cases.
 	if getIPFamilyPolicy(service) == api.IPFamilyPolicySingleStack {
@@ -916,23 +934,10 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
 	}
 
-	// Infer IPFamilyPolicy from IPFamilies[].  This block does not handle the
-	// final defaulting - that happens a bit later, after special cases.
-	if service.Spec.IPFamilyPolicy == nil && len(service.Spec.IPFamilies) == 2 {
-		requireDualStack := api.IPFamilyPolicyRequireDualStack
-		service.Spec.IPFamilyPolicy = &requireDualStack
-	}
-
 	// Special-case: headless + selectorless.  This has to happen before other
 	// checks because it explicitly allows combinations of inputs that would
 	// otherwise be errors.
-	if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone && len(service.Spec.Selector) == 0 {
-		// If the use said nothing about policy and we can't infer it, they get dual-stack
-		if service.Spec.IPFamilyPolicy == nil {
-			requireDualStack := api.IPFamilyPolicyRequireDualStack
-			service.Spec.IPFamilyPolicy = &requireDualStack
-		}
-
+	if service.Spec.ClusterIP == api.ClusterIPNone && len(service.Spec.Selector) == 0 {
 		// If IPFamilies was not set by the user, start with the default
 		// family.
 		if len(service.Spec.IPFamilies) == 0 {
@@ -981,13 +986,6 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
 	}
 
-	// Finally, if IPFamilyPolicy is *still* not set, we can default it to
-	// SingleStack. If there are any webhooks, they have already run.
-	if service.Spec.IPFamilyPolicy == nil {
-		singleStack := api.IPFamilyPolicySingleStack
-		service.Spec.IPFamilyPolicy = &singleStack
-	}
-
 	// nil families, gets cluster default
 	if len(service.Spec.IPFamilies) == 0 {
 		service.Spec.IPFamilies = []api.IPFamily{al.defaultServiceIPFamily}
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index f42093322ef..136826f37d6 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -349,6 +349,7 @@ func TestServiceRegistryUpdateUnspecifiedAllocations(t *testing.T) {
 		create: svctest.MakeService("foo",
 			svctest.SetTypeLoadBalancer,
 			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 			svctest.SetClusterIPs("1.2.3.4", "2000::1"),
 			svctest.SetPorts(
 				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
@@ -372,6 +373,7 @@ func TestServiceRegistryUpdateUnspecifiedAllocations(t *testing.T) {
 		create: svctest.MakeService("foo",
 			svctest.SetTypeLoadBalancer,
 			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 			svctest.SetClusterIPs("1.2.3.4", "2000::1"),
 			svctest.SetPorts(
 				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index fd26a8408a3..4040898d9cf 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	stdruntime "runtime"
 	"strings"
 	"testing"
 
@@ -770,10 +771,25 @@ func TestCreateInitClusterIPsFromClusterIP(t *testing.T) {
 	}
 }
 
+// line returns the line number of the caller, if possible.  This is useful in
+// tests with a large number of cases - when something goes wrong you can find
+// which case more easily.
+func line() string {
+	_, _, line, ok := stdruntime.Caller(1)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%d", line)
+	} else {
+		s = "<??>"
+	}
+	return s
+}
+
 // Prove that create initializes IPFamily fields correctly.
 func TestCreateInitIPFields(t *testing.T) {
 	type testCase struct {
 		name           string
+		line           string
 		svc            *api.Service
 		expectError    bool
 		expectPolicy   api.IPFamilyPolicyType
@@ -796,38 +812,45 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					line:           line(),
 					svc:            svctest.MakeService("foo"),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -835,30 +858,35 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -866,47 +894,55 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
@@ -917,12 +953,14 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v4_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1")),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -930,24 +968,28 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -955,6 +997,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -963,6 +1006,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -970,6 +1014,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -977,6 +1022,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -984,6 +1030,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -991,6 +1038,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -999,6 +1047,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1006,6 +1055,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1013,6 +1063,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1020,12 +1071,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1033,6 +1086,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1040,6 +1094,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1047,6 +1102,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1058,41 +1114,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1100,6 +1163,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1107,6 +1171,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1114,6 +1179,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1121,12 +1187,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1134,6 +1202,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1141,6 +1210,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1148,6 +1218,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1155,12 +1226,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1168,6 +1241,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1175,6 +1249,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1182,6 +1257,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1193,41 +1269,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1235,6 +1318,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1242,6 +1326,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1249,6 +1334,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1256,12 +1342,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1269,6 +1357,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1276,6 +1365,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1283,6 +1373,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1290,12 +1381,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1303,6 +1396,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1310,6 +1404,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1317,6 +1412,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1328,6 +1424,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "Headless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
@@ -1335,6 +1432,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -1343,24 +1441,28 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -1369,6 +1471,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1378,6 +1481,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1385,6 +1489,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1392,6 +1497,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1399,6 +1505,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -1407,6 +1514,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1416,6 +1524,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1423,6 +1532,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1430,6 +1540,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1437,12 +1548,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1450,6 +1563,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1457,6 +1571,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1464,6 +1579,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1475,6 +1591,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil)),
@@ -1483,6 +1600,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1492,6 +1610,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1501,6 +1620,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1510,6 +1630,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1519,6 +1640,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1528,6 +1650,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1538,6 +1661,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1548,6 +1672,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1556,6 +1681,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1564,6 +1690,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1573,6 +1700,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1583,6 +1711,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1593,6 +1722,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1603,6 +1733,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1613,6 +1744,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1622,6 +1754,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1632,6 +1765,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1642,6 +1776,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1652,6 +1787,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -1671,44 +1807,52 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					line:           line(),
 					svc:            svctest.MakeService("foo"),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -1716,30 +1860,35 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -1747,41 +1896,48 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
@@ -1792,18 +1948,21 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v6_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1")),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -1811,18 +1970,21 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -1830,6 +1992,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1837,6 +2000,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1845,6 +2009,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1852,6 +2017,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1859,6 +2025,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -1866,6 +2033,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1873,6 +2041,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1881,6 +2050,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1888,6 +2058,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -1895,12 +2066,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1908,6 +2081,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1915,6 +2089,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1922,6 +2097,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -1933,41 +2109,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1975,6 +2158,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1982,6 +2166,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1989,6 +2174,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -1996,12 +2182,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2009,6 +2197,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2016,6 +2205,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2023,6 +2213,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2030,12 +2221,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2043,6 +2236,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2050,6 +2244,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2057,6 +2252,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2068,41 +2264,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2110,6 +2313,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2117,6 +2321,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2124,6 +2329,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2131,12 +2337,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2144,6 +2352,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2151,6 +2360,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2158,6 +2368,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2165,12 +2376,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2178,6 +2391,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2185,6 +2399,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2192,6 +2407,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2203,6 +2419,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "Headless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
@@ -2210,12 +2427,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -2224,18 +2443,21 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -2244,6 +2466,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2251,6 +2474,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2260,6 +2484,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2267,6 +2492,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2274,6 +2500,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -2282,6 +2509,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2289,6 +2517,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2298,6 +2527,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2305,6 +2535,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2312,12 +2543,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2325,6 +2558,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2332,6 +2566,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2339,6 +2574,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2350,6 +2586,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil)),
@@ -2358,6 +2595,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2367,6 +2605,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2376,6 +2615,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2385,6 +2625,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2394,6 +2635,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2403,6 +2645,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2413,6 +2656,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2423,6 +2667,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2431,6 +2676,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2439,6 +2685,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2448,6 +2695,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2458,6 +2706,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2468,6 +2717,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2478,6 +2728,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2488,6 +2739,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2497,6 +2749,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2507,6 +2760,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2517,6 +2771,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2527,6 +2782,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -2546,41 +2802,46 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					line:           line(),
 					svc:            svctest.MakeService("foo"),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -2588,6 +2849,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -2595,24 +2857,28 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -2620,6 +2886,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -2627,6 +2894,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
@@ -2634,6 +2902,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
@@ -2641,12 +2910,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -2654,6 +2925,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -2661,6 +2933,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
@@ -2668,6 +2941,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
@@ -2679,12 +2953,14 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v4_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1")),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -2692,25 +2968,28 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -2718,6 +2997,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2726,6 +3006,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2733,6 +3014,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2740,6 +3022,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2747,6 +3030,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -2754,6 +3038,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2762,6 +3047,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2769,6 +3055,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2777,6 +3064,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2784,6 +3072,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -2791,6 +3080,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2799,6 +3089,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2806,6 +3097,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2814,6 +3106,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2825,18 +3118,21 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v6_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1")),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -2844,19 +3140,21 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -2864,6 +3162,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2871,6 +3170,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2879,6 +3179,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2886,6 +3187,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -2893,6 +3195,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -2900,6 +3203,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2907,6 +3211,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2915,6 +3220,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2922,6 +3228,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -2930,6 +3237,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -2937,6 +3245,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2944,6 +3253,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2952,6 +3262,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2959,6 +3270,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -2971,44 +3283,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3016,6 +3332,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3023,6 +3340,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3030,6 +3348,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3037,6 +3356,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -3044,6 +3364,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3052,6 +3373,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3059,6 +3381,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3067,6 +3390,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3074,6 +3398,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -3081,6 +3406,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3089,6 +3415,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3096,6 +3423,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3104,6 +3432,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3115,44 +3444,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3160,6 +3493,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3167,6 +3501,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3174,6 +3509,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3181,6 +3517,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -3188,6 +3525,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3195,6 +3533,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3203,6 +3542,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3210,6 +3550,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3218,6 +3559,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -3225,6 +3567,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3232,6 +3575,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3240,6 +3584,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3247,6 +3592,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3259,6 +3605,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "Headless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
@@ -3266,6 +3613,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -3274,6 +3622,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -3282,22 +3631,21 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-					expectHeadless: true,
+					expectError: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-					expectHeadless: true,
+					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -3306,6 +3654,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3315,6 +3664,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3324,6 +3674,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3331,6 +3682,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3338,6 +3690,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -3346,6 +3699,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3355,6 +3709,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3364,6 +3719,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3373,6 +3729,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3382,6 +3739,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -3390,6 +3748,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3399,6 +3758,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3408,6 +3768,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3417,6 +3778,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3430,6 +3792,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil)),
@@ -3438,6 +3801,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3447,6 +3811,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3456,6 +3821,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3465,6 +3831,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3474,6 +3841,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3483,6 +3851,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3493,6 +3862,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3503,6 +3873,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3511,6 +3882,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3519,6 +3891,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3528,6 +3901,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3538,6 +3912,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3548,6 +3923,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3558,6 +3934,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3568,6 +3945,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3577,6 +3955,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3587,6 +3966,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3597,6 +3977,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3607,6 +3988,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -3626,41 +4008,46 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name:           "ClusterIPs:unset_Policy:unset_Families:unset",
+					line:           line(),
 					svc:            svctest.MakeService("foo"),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -3668,6 +4055,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -3675,24 +4063,28 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
 					expectPolicy:   api.IPFamilyPolicyPreferDualStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -3700,6 +4092,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -3707,6 +4100,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
@@ -3714,6 +4108,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
@@ -3721,12 +4116,14 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
 					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -3734,6 +4131,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -3741,6 +4139,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
@@ -3748,6 +4147,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:unset_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
@@ -3759,12 +4159,14 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v4_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1")),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -3772,25 +4174,28 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -3798,6 +4203,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3806,6 +4212,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3813,6 +4220,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3820,6 +4228,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3827,6 +4236,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -3834,6 +4244,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3842,6 +4253,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3849,6 +4261,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3857,6 +4270,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3864,6 +4278,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -3871,6 +4286,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3879,6 +4295,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3886,6 +4303,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3894,6 +4312,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -3905,18 +4324,21 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v6_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1")),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -3924,19 +4346,21 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -3944,6 +4368,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3951,6 +4376,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3959,6 +4385,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3966,6 +4393,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -3973,6 +4401,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -3980,6 +4409,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3987,6 +4417,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -3995,6 +4426,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4002,6 +4434,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4010,6 +4443,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -4017,6 +4451,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4024,6 +4459,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4032,6 +4468,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4039,6 +4476,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4051,44 +4489,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v4v6_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1")),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4096,6 +4538,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4103,6 +4546,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4110,6 +4554,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4117,6 +4562,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -4124,6 +4570,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4132,6 +4579,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4139,6 +4587,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4147,6 +4596,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4154,6 +4604,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -4161,6 +4612,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4169,6 +4621,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4176,6 +4629,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4184,6 +4638,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
 				}, {
 					name: "ClusterIPs:v4v6_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("10.0.0.1", "2000::1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4195,44 +4650,48 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "ClusterIPs:v6v4_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1")),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
+					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4240,6 +4699,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4247,6 +4707,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4254,6 +4715,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4261,6 +4723,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -4268,6 +4731,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4275,6 +4739,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4283,6 +4748,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4290,6 +4756,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4298,6 +4765,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -4305,6 +4773,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4312,6 +4781,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4320,6 +4790,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4327,6 +4798,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "ClusterIPs:v6v4_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetClusterIPs("2000::1", "10.0.0.1"),
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4339,6 +4811,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "Headless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless),
 					expectPolicy:   api.IPFamilyPolicySingleStack,
@@ -4346,6 +4819,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol)),
@@ -4354,6 +4828,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol)),
@@ -4362,22 +4837,21 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol},
-					expectHeadless: true,
+					expectError: true,
 				}, {
 					name: "Headless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-					expectPolicy:   api.IPFamilyPolicyRequireDualStack,
-					expectFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol},
-					expectHeadless: true,
+					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -4386,6 +4860,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4395,6 +4870,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4404,6 +4880,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4411,6 +4888,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -4418,6 +4896,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack)),
@@ -4426,6 +4905,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4435,6 +4915,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4444,6 +4925,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4453,6 +4935,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
@@ -4462,6 +4945,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack)),
@@ -4470,6 +4954,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4479,6 +4964,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4488,6 +4974,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4497,6 +4984,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "Headless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -4510,6 +4998,7 @@ func TestCreateInitIPFields(t *testing.T) {
 				//----------------------------------------
 				{
 					name: "HeadlessSelectorless_Policy:unset_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil)),
@@ -4518,6 +5007,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4527,6 +5017,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4536,6 +5027,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4545,6 +5037,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:unset_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4554,6 +5047,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4563,6 +5057,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4573,6 +5068,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4583,6 +5079,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4591,6 +5088,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:SingleStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4599,6 +5097,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectError: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4608,6 +5107,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4618,6 +5118,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4628,6 +5129,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4638,6 +5140,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:PreferDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4648,6 +5151,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:unset",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4657,6 +5161,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4667,6 +5172,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4677,6 +5183,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v4v6",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4687,6 +5194,7 @@ func TestCreateInitIPFields(t *testing.T) {
 					expectHeadless: true,
 				}, {
 					name: "HeadlessSelectorless_Policy:RequireDualStack_Families:v6v4",
+					line: line(),
 					svc: svctest.MakeService("foo",
 						svctest.SetHeadless,
 						svctest.SetSelector(nil),
@@ -4712,7 +5220,7 @@ func TestCreateInitIPFields(t *testing.T) {
 			defer storage.Store.DestroyFunc()
 
 			for _, itc := range otc.cases {
-				t.Run(itc.name, func(t *testing.T) {
+				t.Run(itc.name+"__@L"+itc.line, func(t *testing.T) {
 					ctx := genericapirequest.NewDefaultContext()
 					createdObj, err := storage.Create(ctx, itc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
 					if itc.expectError && err != nil {
@@ -4805,7 +5313,7 @@ func TestCreateInvalidClusterIPInputs(t *testing.T) {
 		families: []api.IPFamily{api.IPv4Protocol},
 		svc: svctest.MakeService("foo",
 			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-		expect: []string{"not configured on this cluster"},
+		expect: []string{"when multiple IP families are specified"},
 	}, {
 		name:     "dup_ipFamily_singlestack",
 		families: []api.IPFamily{api.IPv4Protocol},
@@ -4882,7 +5390,6 @@ func TestCreateInvalidClusterIPInputs(t *testing.T) {
 			if err == nil {
 				t.Fatalf("unexpected success creating service")
 			}
-			t.Logf("INFO: errstr:\n  %v", err)
 			for _, s := range tc.expect {
 				if !strings.Contains(err.Error(), s) {
 					t.Errorf("expected to find %q in the error:\n  %s", s, err.Error())
@@ -4897,13 +5404,17 @@ func TestCreateDeleteReuse(t *testing.T) {
 		svc  *api.Service
 	}{{
 		name: "v4",
-		svc:  svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetIPFamilies(api.IPv4Protocol)),
+		svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+			svctest.SetIPFamilies(api.IPv4Protocol)),
 	}, {
 		name: "v6",
-		svc:  svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetIPFamilies(api.IPv6Protocol)),
+		svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+			svctest.SetIPFamilies(api.IPv6Protocol)),
 	}, {
 		name: "v4v6",
-		svc:  svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
+		svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+			svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
 	}}
 
 	// This test is ONLY with the gate enabled.
@@ -5975,8 +6486,8 @@ func TestUpdateDryRun(t *testing.T) {
 	}
 }
 
-// Proves that updates from single-stack to dual-stack work.
-func TestUpdateSingleToDualStack(t *testing.T) {
+// Proves that updates from single-stack work.
+func TestUpdateIPsFromSingleStack(t *testing.T) {
 	prove := func(proofs ...svcTestProof) []svcTestProof {
 		return proofs
 	}
@@ -5992,6 +6503,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 	// Single-stack cases as control.
 	testCasesV4 := []cudTestCase{{
 		name: "single-single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -6007,6 +6519,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -6021,6 +6534,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_policy",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -6034,6 +6548,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_families",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack)),
@@ -6057,6 +6572,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 	//    ips={nil, single, dual}
 	testCasesV4V6 := []cudTestCase{{
 		name: "policy:nil_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6072,6 +6588,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6088,6 +6605,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6099,11 +6617,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6120,6 +6638,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6137,6 +6656,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6149,11 +6669,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv4Protocol),
 				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6165,28 +6685,27 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol)),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
 				svctest.SetClusterIPs("10.0.0.1")),
 			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(1)),
 		},
 		update: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
 				svctest.SetClusterIPs("10.0.0.1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6199,11 +6718,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv4Protocol, api.IPv6Protocol),
 				svctest.SetClusterIPs("10.0.0.1", "2000::1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:single_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6220,6 +6739,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6237,6 +6757,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6253,6 +6774,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6270,6 +6792,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6288,6 +6811,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6305,6 +6829,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6321,6 +6846,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6338,6 +6864,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6355,6 +6882,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6371,6 +6899,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6388,6 +6917,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6405,6 +6935,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6422,6 +6953,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6440,6 +6972,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6458,6 +6991,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6475,6 +7009,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6493,6 +7028,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6511,6 +7047,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6527,6 +7064,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6544,6 +7082,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6561,6 +7100,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6578,6 +7118,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6596,6 +7137,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6614,6 +7156,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6631,6 +7174,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6649,6 +7193,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6667,6 +7212,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_wrong_order_families",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6682,6 +7228,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_wrong_order_ips",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6697,6 +7244,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_ip_in_use",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6731,6 +7279,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 	//    ips={nil, single, dual}
 	testCasesV6V4 := []cudTestCase{{
 		name: "policy:nil_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6746,6 +7295,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6762,6 +7312,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6773,11 +7324,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6794,6 +7345,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6811,6 +7363,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6823,11 +7376,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv6Protocol),
 				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6839,11 +7392,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol)),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6856,11 +7409,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
 				svctest.SetClusterIPs("2000::1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6873,11 +7426,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies(api.IPv6Protocol, api.IPv4Protocol),
 				svctest.SetClusterIPs("2000::1", "10.0.0.1")),
-			expectClusterIPs: true,
-			prove:            prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:single_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6894,6 +7447,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6911,6 +7465,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6927,6 +7482,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6944,6 +7500,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6962,6 +7519,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6979,6 +7537,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -6995,6 +7554,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7012,6 +7572,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7029,6 +7590,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7045,6 +7607,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7062,6 +7625,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7079,6 +7643,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7096,6 +7661,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7114,6 +7680,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7132,6 +7699,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7149,6 +7717,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7167,6 +7736,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7185,6 +7755,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7201,6 +7772,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7218,6 +7790,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7235,6 +7808,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7252,6 +7826,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7270,6 +7845,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7288,6 +7864,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7305,6 +7882,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7323,6 +7901,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7341,6 +7920,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_wrong_order_families",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7356,6 +7936,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_wrong_order_ips",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7371,6 +7952,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "single-dual_ip_in_use",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7404,6 +7986,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 	//    families={nil, single, dual}
 	testCasesHeadless := []cudTestCase{{
 		name: "policy:nil_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7419,6 +8002,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7435,6 +8019,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7446,11 +8031,11 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetSelector(map[string]string{"k2": "v2"}),
 				svctest.SetIPFamilies("IPv4", "IPv6")),
-			expectHeadless: true,
-			prove:          prove(proveNumFamilies(2)),
+			expectError: true,
 		},
 	}, {
 		name: "policy:single_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7467,6 +8052,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7484,6 +8070,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7500,6 +8087,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7516,6 +8104,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7533,6 +8122,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7550,6 +8140,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7566,6 +8157,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7583,6 +8175,7 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicySingleStack),
@@ -7605,8 +8198,8 @@ func TestUpdateSingleToDualStack(t *testing.T) {
 	})
 }
 
-// Proves that updates from dual-stack to single-stack work.
-func TestUpdateDualToSingleStack(t *testing.T) {
+// Proves that updates from dual-stack.
+func TestUpdateIPsFromDualStack(t *testing.T) {
 	prove := func(proofs ...svcTestProof) []svcTestProof {
 		return proofs
 	}
@@ -7625,6 +8218,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 	//    ips={nil, single, dual}
 	testCasesV4V6 := []cudTestCase{{
 		name: "policy:nil_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7640,6 +8234,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7655,6 +8250,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7671,6 +8267,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7686,6 +8283,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7702,6 +8300,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7718,6 +8317,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7734,6 +8334,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7750,6 +8351,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7767,6 +8369,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7783,6 +8386,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7800,6 +8404,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7818,6 +8423,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7835,6 +8441,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7853,6 +8460,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7872,6 +8480,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7890,6 +8499,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7909,6 +8519,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7928,6 +8539,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7944,6 +8556,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7960,6 +8573,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7977,6 +8591,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -7993,6 +8608,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8010,6 +8626,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8027,6 +8644,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8044,6 +8662,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8061,6 +8680,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8079,6 +8699,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8095,6 +8716,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8111,6 +8733,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8128,6 +8751,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8144,6 +8768,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8161,6 +8786,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8178,6 +8804,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8195,6 +8822,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8212,6 +8840,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8230,6 +8859,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "dual-single_wrong_order_families",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8245,6 +8875,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "dual-single_wrong_order_ips",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8270,6 +8901,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 	//    ips={nil, single, dual}
 	testCasesV6V4 := []cudTestCase{{
 		name: "policy:nil_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8285,6 +8917,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8300,6 +8933,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8316,6 +8950,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8331,6 +8966,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8347,6 +8983,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8363,6 +9000,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8379,6 +9017,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8395,6 +9034,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8412,6 +9052,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8428,6 +9069,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8445,6 +9087,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8463,6 +9106,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8480,6 +9124,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8498,6 +9143,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8517,6 +9163,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8535,6 +9182,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8554,6 +9202,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8573,6 +9222,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8589,6 +9239,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8605,6 +9256,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8622,6 +9274,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8638,6 +9291,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8655,6 +9309,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8672,6 +9327,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8689,6 +9345,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8706,6 +9363,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8724,6 +9382,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8740,6 +9399,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8756,6 +9416,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8773,6 +9434,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8789,6 +9451,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8806,6 +9469,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8823,6 +9487,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8840,6 +9505,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8857,6 +9523,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual_ips:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8875,6 +9542,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "dual-single_wrong_order_families",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8890,6 +9558,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "dual-single_wrong_order_ips",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8914,6 +9583,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 	//    families={nil, single, dual}
 	testCasesHeadless := []cudTestCase{{
 		name: "policy:nil_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8925,10 +9595,11 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetSelector(map[string]string{"k2": "v2"})),
 			expectHeadless: true,
-			prove:          prove(proveNumFamilies(1)),
+			prove:          prove(proveNumFamilies(2)),
 		},
 	}, {
 		name: "policy:nil_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8944,6 +9615,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:nil_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8960,6 +9632,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8976,6 +9649,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -8993,6 +9667,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:single_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -9011,6 +9686,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -9027,6 +9703,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -9043,6 +9720,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:prefer_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -9060,6 +9738,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:nil",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -9076,6 +9755,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:single",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -9092,6 +9772,7 @@ func TestUpdateDualToSingleStack(t *testing.T) {
 		},
 	}, {
 		name: "policy:require_families:dual",
+		line: line(),
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
 				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyRequireDualStack),
@@ -9316,6 +9997,7 @@ func proveHealthCheckNodePortDeallocated(t *testing.T, storage *GenericREST, bef
 
 type cudTestCase struct {
 	name         string
+	line         string // if not empty, will be logged with errors, use line() to set
 	create       svcTestCase
 	beforeUpdate func(t *testing.T, storage *GenericREST)
 	update       svcTestCase
@@ -9338,7 +10020,11 @@ func helpTestCreateUpdateDeleteWithFamilies(t *testing.T, testCases []cudTestCas
 	defer storage.Store.DestroyFunc()
 
 	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
+		name := tc.name
+		if tc.line != "" {
+			name += "__@L" + tc.line
+		}
+		t.Run(name, func(t *testing.T) {
 			ctx := genericapirequest.NewDefaultContext()
 
 			// Create the object as specified and check the results.
@@ -9772,11 +10458,13 @@ func TestFeatureClusterIPs(t *testing.T) {
 		name: "clusterIPs:valid-valid",
 		create: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 				svctest.SetClusterIPs("10.0.0.93", "2000::93")),
 			expectClusterIPs: true,
 		},
 		update: svcTestCase{
 			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
 				svctest.SetClusterIPs("10.0.0.76", "2000::76")),
 			expectError: true,
 		},
diff --git a/test/e2e/network/dual_stack.go b/test/e2e/network/dual_stack.go
index de9f945dd5e..2941ac037ee 100644
--- a/test/e2e/network/dual_stack.go
+++ b/test/e2e/network/dual_stack.go
@@ -367,7 +367,7 @@ var _ = common.SIGDescribe("[Feature:IPv6DualStack]", func() {
 		expectedPolicy := v1.IPFamilyPolicyRequireDualStack
 		expectedFamilies := []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}
 
-		service := createService(t.ServiceName, t.Namespace, t.Labels, nil, expectedFamilies)
+		service := createService(t.ServiceName, t.Namespace, t.Labels, &expectedPolicy, expectedFamilies)
 
 		jig.Labels = t.Labels
 		err := jig.CreateServicePods(2)
@@ -412,7 +412,7 @@ var _ = common.SIGDescribe("[Feature:IPv6DualStack]", func() {
 		expectedPolicy := v1.IPFamilyPolicyRequireDualStack
 		expectedFamilies := []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}
 
-		service := createService(t.ServiceName, t.Namespace, t.Labels, nil, expectedFamilies)
+		service := createService(t.ServiceName, t.Namespace, t.Labels, &expectedPolicy, expectedFamilies)
 
 		jig.Labels = t.Labels
 		err := jig.CreateServicePods(2)

From cf4804643ad9d0218f9ecef1da5274adcb1ac6ab Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 5 Dec 2020 16:33:42 -0800
Subject: [PATCH 44/79] Svc REST: Remove obviously unused args

This is part of the de-layering conclusion and cleanup.

Bridge the old tests into the new REST.  This will all be removed soon.
---
 pkg/registry/core/rest/storage_core.go        |  4 ----
 pkg/registry/core/service/storage/rest.go     | 13 -----------
 .../core/service/storage/rest_test.go         | 23 ++-----------------
 3 files changed, 2 insertions(+), 38 deletions(-)

diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go
index 443a955a8d8..5f21257dc9e 100644
--- a/pkg/registry/core/rest/storage_core.go
+++ b/pkg/registry/core/rest/storage_core.go
@@ -275,11 +275,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 
 	serviceRest, serviceRestProxy := servicestore.NewREST(
 		serviceRESTStorage,
-		endpointsStorage,
-		podStorage.Pod,
 		serviceClusterIPAllocator.IPFamily(),
-		serviceIPAllocators,
-		serviceNodePortAllocator,
 		c.ProxyTransport)
 
 	restStorageMap := map[string]rest.Storage{
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index c0dbf401f6f..adbc2766119 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -46,10 +46,7 @@ import (
 
 // REST adapts a service registry into apiserver's RESTStorage model.
 type REST struct {
-	strategy       rest.RESTCreateUpdateStrategy
 	services       ServiceStorage
-	endpoints      EndpointsStorage
-	alloc          RESTAllocStuff
 	proxyTransport http.RoundTripper
 	pods           rest.Getter
 }
@@ -90,25 +87,15 @@ type ServiceStorage interface {
 //   or the strategy.
 func NewREST(
 	services ServiceStorage,
-	endpoints EndpointsStorage,
-	pods rest.Getter,
 	defaultFamily api.IPFamily,
-	ipAllocs map[api.IPFamily]ipallocator.Interface,
-	portAlloc portallocator.Interface,
 	proxyTransport http.RoundTripper,
 ) (*REST, *registry.ProxyREST) {
 
-	strategy, _ := registry.StrategyForServiceCIDRs(ipAllocs[defaultFamily].CIDR(), len(ipAllocs) > 1)
-
 	klog.V(0).Infof("the default service ipfamily for this cluster is: %s", string(defaultFamily))
 
 	rest := &REST{
-		strategy:       strategy,
 		services:       services,
-		endpoints:      endpoints,
 		proxyTransport: proxyTransport,
-		pods:           pods,
-		alloc:          makeAlloc(defaultFamily, ipAllocs, portAlloc),
 	}
 
 	return rest, &registry.ProxyREST{Redirector: rest, ProxyTransport: proxyTransport}
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 136826f37d6..53703a2b1b8 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -42,7 +42,6 @@ import (
 	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	endpointstore "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
-	podstore "k8s.io/kubernetes/pkg/registry/core/pod/storage"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
@@ -187,25 +186,6 @@ func (s *serviceStorage) ResourceLocation(ctx context.Context, id string) (remot
 func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 
-	podStorage, err := podstore.NewStorage(generic.RESTOptions{
-		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "pods"}),
-		Decorator:               generic.UndecoratedStorage,
-		DeleteCollectionWorkers: 3,
-		ResourcePrefix:          "pods",
-	}, nil, nil, nil)
-	if err != nil {
-		t.Fatalf("unexpected error from REST storage: %v", err)
-	}
-
-	endpointStorage, err := endpointstore.NewREST(generic.RESTOptions{
-		StorageConfig:  etcdStorage.ForResource(schema.GroupResource{Resource: "endpoints"}),
-		Decorator:      generic.UndecoratedStorage,
-		ResourcePrefix: "endpoints",
-	})
-	if err != nil {
-		t.Fatalf("unexpected error from REST storage: %v", err)
-	}
-
 	var rPrimary ipallocator.Interface
 	var rSecondary ipallocator.Interface
 
@@ -214,6 +194,7 @@ func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.
 	}
 	for i, family := range ipFamilies {
 		var r ipallocator.Interface
+		var err error
 		switch family {
 		case api.IPv4Protocol:
 			r, err = ipallocator.NewInMemory(makeIPNet(t))
@@ -248,7 +229,7 @@ func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.
 	}
 
 	inner := newInnerREST(t, etcdStorage, ipAllocators, portAllocator)
-	rest, _ := NewREST(inner, endpointStorage, podStorage.Pod, rPrimary.IPFamily(), ipAllocators, portAllocator, nil)
+	rest, _ := NewREST(inner, rPrimary.IPFamily(), nil)
 
 	return rest, server
 }

From 8e68b587e8755b3aacdec28f343a7c9b0e7e5493 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 5 Dec 2020 16:37:29 -0800
Subject: [PATCH 45/79] Svc REST: De-layering done! Convert to 1 layer

This is the culmination of all the previous commits which made this last
move less dramatic.  More tests and cleanup commits will follow.

Background, for future archaeologists:

Service has (had) an "outer" and "inner" REST handler.  This is because of how we do IP and port allocations synchronously, but since we don't have API transactions, we need to roll those back in case of a failure.  Both layers use the same `Strategy`, but the outer calls into the inner, which causes a lot of complexity in the code (including an open-coded partial reimplementation of a date-unknown snapshot of the generic REST code) and results in `Prepare` and `Validate` hooks being called twice.

The "normal" REST flow seems to be:

```
mutating webhooks
generic REST store Create {
    cleanup = BeginCreate
    BeforeCreate {
        strategy.PrepareForCreate {
            dropDisabledFields
        }
        strategy.Validate
        strategy.Canonicalize
    }
    createValidation (validating webhooks)
    storage Create
    cleanup
    AfterCreate
    Decorator
}
```

Service (before this series of commits) does:

```
mutating webhooks
svc custom Create {
    BeforeCreate {
        strategy.PrepareForCreate {
            dropDisabledFields
        }
        strategy.Validate
        strategy.Canonicalize
    }
    Allocations
    inner (generic) Create {
        cleanup = BeginCreate
        BeforeCreate {
            strategy.PrepareForCreate {
                dropDisabledFields
            }
            strategy.Validate
            strategy.Canonicalize
        }
        createValidation (validating webhooks)
        storage Create
        cleanup
        AfterCreate
        Decorator
    }
}
```

After this:

```
mutating webhooks
generic REST store Create {
    cleanup = BeginCreate
        Allocations
    BeforeCreate {
        strategy.PrepareForCreate {
            dropDisabledFields
        }
        strategy.Validate
        strategy.Canonicalize
    }
    createValidation (validating webhooks)
    storage Create
    cleanup
    AfterCreate
        Rollback allocations on error
    Decorator
}
```
---
 pkg/registry/core/rest/storage_core.go        |  11 +-
 pkg/registry/core/service/storage/rest.go     |  21 ---
 .../core/service/storage/rest_test.go         | 160 +-----------------
 3 files changed, 7 insertions(+), 185 deletions(-)

diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go
index 5f21257dc9e..7886ab2ddaf 100644
--- a/pkg/registry/core/rest/storage_core.go
+++ b/pkg/registry/core/rest/storage_core.go
@@ -261,7 +261,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		serviceIPAllocators[secondaryServiceClusterIPAllocator.IPFamily()] = secondaryServiceClusterIPAllocator
 	}
 
-	serviceRESTStorage, serviceStatusStorage, _, err := servicestore.NewGenericREST(
+	serviceRESTStorage, serviceStatusStorage, serviceRESTProxy, err := servicestore.NewGenericREST(
 		restOptionsGetter,
 		serviceClusterIPAllocator.IPFamily(),
 		serviceIPAllocators,
@@ -273,11 +273,6 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err
 	}
 
-	serviceRest, serviceRestProxy := servicestore.NewREST(
-		serviceRESTStorage,
-		serviceClusterIPAllocator.IPFamily(),
-		c.ProxyTransport)
-
 	restStorageMap := map[string]rest.Storage{
 		"pods":             podStorage.Pod,
 		"pods/attach":      podStorage.Attach,
@@ -294,8 +289,8 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		"replicationControllers":        controllerStorage.Controller,
 		"replicationControllers/status": controllerStorage.Status,
 
-		"services":        serviceRest,
-		"services/proxy":  serviceRestProxy,
+		"services":        serviceRESTStorage,
+		"services/proxy":  serviceRESTProxy,
 		"services/status": serviceStatusStorage,
 
 		"endpoints": endpointsStorage,
diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index adbc2766119..853dd0316d8 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -37,7 +37,6 @@ import (
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/features"
-	registry "k8s.io/kubernetes/pkg/registry/core/service"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	netutils "k8s.io/utils/net"
@@ -81,26 +80,6 @@ type ServiceStorage interface {
 	rest.Redirector
 }
 
-// NewREST returns a wrapper around the underlying generic storage and performs
-// allocations and deallocations of various service related resources like ports.
-// TODO: all transactional behavior should be supported from within generic storage
-//   or the strategy.
-func NewREST(
-	services ServiceStorage,
-	defaultFamily api.IPFamily,
-	proxyTransport http.RoundTripper,
-) (*REST, *registry.ProxyREST) {
-
-	klog.V(0).Infof("the default service ipfamily for this cluster is: %s", string(defaultFamily))
-
-	rest := &REST{
-		services:       services,
-		proxyTransport: proxyTransport,
-	}
-
-	return rest, &registry.ProxyREST{Redirector: rest, ProxyTransport: proxyTransport}
-}
-
 // This is a trasitionary function to facilitate service REST flattening.
 func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) RESTAllocStuff {
 	return RESTAllocStuff{
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 53703a2b1b8..e1bdc3e1997 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -17,28 +17,18 @@ limitations under the License.
 package storage
 
 import (
-	"context"
-	"fmt"
 	"net"
-	"net/http"
-	"net/url"
 	"reflect"
-	"sort"
 	"testing"
 
-	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/apimachinery/pkg/watch"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
-	"k8s.io/apiserver/pkg/storage/storagebackend"
-	"k8s.io/apiserver/pkg/util/dryrun"
 	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	endpointstore "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
@@ -47,143 +37,9 @@ import (
 	"k8s.io/kubernetes/pkg/registry/registrytest"
 	netutils "k8s.io/utils/net"
 	utilpointer "k8s.io/utils/pointer"
-	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
-// TODO(wojtek-t): Cleanup this file.
-// It is now testing mostly the same things as other resources but
-// in a completely different way. We should unify it.
-
-type serviceStorage struct {
-	inner    *GenericREST
-	Services map[string]*api.Service
-}
-
-func (s *serviceStorage) saveService(svc *api.Service) {
-	if s.Services == nil {
-		s.Services = map[string]*api.Service{}
-	}
-	s.Services[svc.Name] = svc.DeepCopy()
-}
-
-func (s *serviceStorage) NamespaceScoped() bool {
-	return true
-}
-
-func (s *serviceStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
-	if s.Services[name] == nil {
-		return nil, fmt.Errorf("service %q not found", name)
-	}
-	return s.Services[name].DeepCopy(), nil
-}
-
-func getService(getter rest.Getter, ctx context.Context, name string, options *metav1.GetOptions) (*api.Service, error) {
-	obj, err := getter.Get(ctx, name, options)
-	if err != nil {
-		return nil, err
-	}
-	return obj.(*api.Service), nil
-}
-
-func (s *serviceStorage) NewList() runtime.Object {
-	panic("not implemented")
-}
-
-func (s *serviceStorage) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {
-	ns, _ := genericapirequest.NamespaceFrom(ctx)
-
-	keys := make([]string, 0, len(s.Services))
-	for k := range s.Services {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	res := new(api.ServiceList)
-	for _, k := range keys {
-		svc := s.Services[k]
-		if ns == metav1.NamespaceAll || ns == svc.Namespace {
-			res.Items = append(res.Items, *svc)
-		}
-	}
-
-	return res, nil
-}
-
-func (s *serviceStorage) New() runtime.Object {
-	panic("not implemented")
-}
-
-func (s *serviceStorage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
-	ret, err := s.inner.Create(ctx, obj, createValidation, options)
-	if err != nil {
-		return ret, err
-	}
-
-	if dryrun.IsDryRun(options.DryRun) {
-		return ret.DeepCopyObject(), nil
-	}
-	svc := ret.(*api.Service)
-	s.saveService(svc)
-
-	return s.Services[svc.Name].DeepCopy(), nil
-}
-
-func (s *serviceStorage) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
-	ret, created, err := s.inner.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
-	if err != nil {
-		return ret, created, err
-	}
-	if dryrun.IsDryRun(options.DryRun) {
-		return ret.DeepCopyObject(), created, err
-	}
-	svc := ret.(*api.Service)
-	s.saveService(svc)
-
-	return s.Services[name].DeepCopy(), created, nil
-}
-
-func (s *serviceStorage) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
-	ret, del, err := s.inner.Delete(ctx, name, deleteValidation, options)
-	if err != nil {
-		return ret, del, err
-	}
-
-	if dryrun.IsDryRun(options.DryRun) {
-		return ret.DeepCopyObject(), del, nil
-	}
-	delete(s.Services, name)
-
-	return ret.DeepCopyObject(), del, err
-}
-
-func (s *serviceStorage) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) {
-	panic("not implemented")
-}
-
-func (s *serviceStorage) Watch(ctx context.Context, options *metainternalversion.ListOptions) (watch.Interface, error) {
-	panic("not implemented")
-}
-
-func (s *serviceStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
-	panic("not implemented")
-}
-
-func (s *serviceStorage) StorageVersion() runtime.GroupVersioner {
-	panic("not implemented")
-}
-
-// GetResetFields implements rest.ResetFieldsStrategy
-func (s *serviceStorage) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
-	//FIXME: should panic?
-	return nil
-}
-
-// ResourceLocation implements rest.Redirector
-func (s *serviceStorage) ResourceLocation(ctx context.Context, id string) (remoteLocation *url.URL, transport http.RoundTripper, err error) {
-	panic("not implemented")
-}
-
-func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.EtcdTestServer) {
+func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 
 	var rPrimary ipallocator.Interface
@@ -228,15 +84,6 @@ func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*REST, *etcd3testing.
 		ipAllocators[rSecondary.IPFamily()] = rSecondary
 	}
 
-	inner := newInnerREST(t, etcdStorage, ipAllocators, portAllocator)
-	rest, _ := NewREST(inner, rPrimary.IPFamily(), nil)
-
-	return rest, server
-}
-
-// This bridges to the "inner" REST implementation so tests continue to run
-// during the delayering of service REST code.
-func newInnerREST(t *testing.T, etcdStorage *storagebackend.ConfigForResource, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) *serviceStorage {
 	restOptions := generic.RESTOptions{
 		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
 		Decorator:               generic.UndecoratedStorage,
@@ -249,11 +96,12 @@ func newInnerREST(t *testing.T, etcdStorage *storagebackend.ConfigForResource, i
 		ResourcePrefix: "endpoints",
 	})
 
-	inner, _, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocs, portAlloc, endpoints, nil, nil)
+	rest, _, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocators, portAllocator, endpoints, nil, nil)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
-	return &serviceStorage{inner: inner}
+
+	return rest, server
 }
 
 func makeIPNet(t *testing.T) *net.IPNet {

From 03e7690cdbbdf6d8dd053f112d8e93214a8d96c5 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 5 Dec 2020 16:42:47 -0800
Subject: [PATCH 46/79] Svc REST: Remove old, now unused stubs

---
 pkg/registry/core/service/storage/rest.go    | 99 --------------------
 pkg/registry/core/service/storage/storage.go | 11 ++-
 2 files changed, 8 insertions(+), 102 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 853dd0316d8..13a2838e0b1 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -19,16 +19,11 @@ package storage
 import (
 	"context"
 	"fmt"
-	"net/http"
-	"net/url"
 
 	"k8s.io/apimachinery/pkg/api/errors"
-	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
-	"k8s.io/apimachinery/pkg/watch"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -40,16 +35,8 @@ import (
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	netutils "k8s.io/utils/net"
-	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
-// REST adapts a service registry into apiserver's RESTStorage model.
-type REST struct {
-	services       ServiceStorage
-	proxyTransport http.RoundTripper
-	pods           rest.Getter
-}
-
 // RESTAllocStuff is a temporary struct to facilitate the flattening of service
 // REST layers.  It will be cleaned up over a series of commits.
 type RESTAllocStuff struct {
@@ -68,18 +55,6 @@ type ServiceNodePort struct {
 	NodePort int32
 }
 
-type ServiceStorage interface {
-	rest.Scoper
-	rest.Getter
-	rest.Lister
-	rest.CreaterUpdater
-	rest.GracefulDeleter
-	rest.Watcher
-	rest.StorageVersionProvider
-	rest.ResetFieldsStrategy
-	rest.Redirector
-}
-
 // This is a trasitionary function to facilitate service REST flattening.
 func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) RESTAllocStuff {
 	return RESTAllocStuff{
@@ -89,55 +64,6 @@ func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator
 	}
 }
 
-var (
-	_ ServiceStorage              = &REST{}
-	_ rest.CategoriesProvider     = &REST{}
-	_ rest.ShortNamesProvider     = &REST{}
-	_ rest.StorageVersionProvider = &REST{}
-)
-
-func (rs *REST) StorageVersion() runtime.GroupVersioner {
-	return rs.services.StorageVersion()
-}
-
-// ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource.
-func (rs *REST) ShortNames() []string {
-	return []string{"svc"}
-}
-
-// Categories implements the CategoriesProvider interface. Returns a list of categories a resource is part of.
-func (rs *REST) Categories() []string {
-	return []string{"all"}
-}
-
-func (rs *REST) NamespaceScoped() bool {
-	return rs.services.NamespaceScoped()
-}
-
-func (rs *REST) New() runtime.Object {
-	return rs.services.New()
-}
-
-func (rs *REST) NewList() runtime.Object {
-	return rs.services.NewList()
-}
-
-func (rs *REST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
-	return rs.services.Get(ctx, name, options)
-}
-
-func (rs *REST) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {
-	return rs.services.List(ctx, options)
-}
-
-func (rs *REST) Watch(ctx context.Context, options *metainternalversion.ListOptions) (watch.Interface, error) {
-	return rs.services.Watch(ctx, options)
-}
-
-func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
-	return rs.services.Create(ctx, obj, createValidation, options)
-}
-
 func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (transaction, error) {
 	result := metaTransaction{}
 
@@ -169,10 +95,6 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 	return result, nil
 }
 
-func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
-	return rs.services.Delete(ctx, id, deleteValidation, options)
-}
-
 func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
 	al.releaseServiceClusterIPs(svc)
 
@@ -237,10 +159,6 @@ func (al *RESTAllocStuff) healthCheckNodePortUpdate(oldService, service *api.Ser
 	return true, nil
 }
 
-func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
-	return rs.services.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
-}
-
 func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRun bool) (transaction, error) {
 	result := metaTransaction{}
 
@@ -314,23 +232,6 @@ func (al *RESTAllocStuff) allocUpdateServiceNodePortsNew(service, oldService *ap
 	return txn, nil
 }
 
-// GetResetFields implements rest.ResetFieldsStrategy
-func (rs *REST) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
-	return rs.services.GetResetFields()
-}
-
-// Implement Redirector.
-var _ = rest.Redirector(&REST{})
-
-// ResourceLocation returns a URL to which one can send traffic for the specified service.
-func (rs *REST) ResourceLocation(ctx context.Context, id string) (*url.URL, http.RoundTripper, error) {
-	return rs.services.ResourceLocation(ctx, id)
-}
-
-func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
-	return r.services.ConvertToTable(ctx, object, tableOptions)
-}
-
 func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
 	allocated := make(map[api.IPFamily]string)
 
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index dbd01c4b097..0a9353df990 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -70,6 +70,14 @@ type GenericREST struct {
 	proxyTransport    http.RoundTripper
 }
 
+var (
+	_ rest.CategoriesProvider     = &GenericREST{}
+	_ rest.ShortNamesProvider     = &GenericREST{}
+	_ rest.StorageVersionProvider = &GenericREST{}
+	_ rest.ResetFieldsStrategy    = &GenericREST{}
+	_ rest.Redirector             = &GenericREST{}
+)
+
 // NewGenericREST returns a RESTStorage object that will work against services.
 func NewGenericREST(
 	optsGetter generic.RESTOptionsGetter,
@@ -371,9 +379,6 @@ func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Objec
 	return finish, nil
 }
 
-// Implement Redirector.
-var _ rest.Redirector = &GenericREST{}
-
 // ResourceLocation returns a URL to which one can send traffic for the specified service.
 func (r *GenericREST) ResourceLocation(ctx context.Context, id string) (*url.URL, http.RoundTripper, error) {
 	// Allow ID as "svcname", "svcname:port", or "scheme:svcname:port".

From 245a654decd5c3f9d3af338d9ca6ae930f5c8869 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 28 Feb 2021 22:00:01 -0800
Subject: [PATCH 47/79] Svc REST: Rename service NewGenericREST to NewREST

Just like all the other registries.
---
 pkg/registry/core/rest/storage_core.go            | 2 +-
 pkg/registry/core/service/storage/rest_test.go    | 2 +-
 pkg/registry/core/service/storage/storage.go      | 4 ++--
 pkg/registry/core/service/storage/storage_test.go | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go
index 7886ab2ddaf..cdb9af6f4ae 100644
--- a/pkg/registry/core/rest/storage_core.go
+++ b/pkg/registry/core/rest/storage_core.go
@@ -261,7 +261,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
 		serviceIPAllocators[secondaryServiceClusterIPAllocator.IPFamily()] = secondaryServiceClusterIPAllocator
 	}
 
-	serviceRESTStorage, serviceStatusStorage, serviceRESTProxy, err := servicestore.NewGenericREST(
+	serviceRESTStorage, serviceStatusStorage, serviceRESTProxy, err := servicestore.NewREST(
 		restOptionsGetter,
 		serviceClusterIPAllocator.IPFamily(),
 		serviceIPAllocators,
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index e1bdc3e1997..d03271af12c 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -96,7 +96,7 @@ func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *etcd3t
 		ResourcePrefix: "endpoints",
 	})
 
-	rest, _, _, err := NewGenericREST(restOptions, api.IPv4Protocol, ipAllocators, portAllocator, endpoints, nil, nil)
+	rest, _, _, err := NewREST(restOptions, api.IPv4Protocol, ipAllocators, portAllocator, endpoints, nil, nil)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 0a9353df990..70f718b0dcc 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -78,8 +78,8 @@ var (
 	_ rest.Redirector             = &GenericREST{}
 )
 
-// NewGenericREST returns a RESTStorage object that will work against services.
-func NewGenericREST(
+// NewREST returns a RESTStorage object that will work against services.
+func NewREST(
 	optsGetter generic.RESTOptionsGetter,
 	serviceIPFamily api.IPFamily,
 	ipAllocs map[api.IPFamily]ipallocator.Interface,
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 4040898d9cf..5bf7a2a0de0 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -154,7 +154,7 @@ func newStorageWithPods(t *testing.T, ipFamilies []api.IPFamily, pods []api.Pod,
 		}
 	}
 
-	serviceStorage, statusStorage, _, err := NewGenericREST(restOptions, ipFamilies[0], ipAllocs, portAlloc, endpointsStorage, podStorage.Pod, nil)
+	serviceStorage, statusStorage, _, err := NewREST(restOptions, ipFamilies[0], ipAllocs, portAlloc, endpointsStorage, podStorage.Pod, nil)
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}

From 652dc8787ca78e13903786870c6b4e62ae2d66ab Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 31 Jul 2021 16:21:16 -0700
Subject: [PATCH 48/79] Svc REST: Use "prove" helpers in other tests

---
 .../core/service/storage/storage_test.go      | 255 +++---------------
 1 file changed, 33 insertions(+), 222 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 5bf7a2a0de0..5399b8169b7 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -5242,22 +5242,10 @@ func TestCreateInitIPFields(t *testing.T) {
 						t.Errorf("wrong IPFamilies: want %s, got %s", want, got)
 					}
 					if itc.expectHeadless {
-						if !reflect.DeepEqual(createdSvc.Spec.ClusterIPs, []string{"None"}) {
-							t.Fatalf("wrong clusterIPs: want [\"None\"], got %v", createdSvc.Spec.ClusterIPs)
-						}
+						proveHeadless(t, storage, nil, createdSvc)
 						return
 					}
-					if c, f := len(createdSvc.Spec.ClusterIPs), len(createdSvc.Spec.IPFamilies); c != f {
-						t.Errorf("clusterIPs and ipFamilies are not the same length: %d vs %d", c, f)
-					}
-					for i, clip := range createdSvc.Spec.ClusterIPs {
-						if cf, ef := familyOf(clip), createdSvc.Spec.IPFamilies[i]; cf != ef {
-							t.Errorf("clusterIP is the wrong IP family: want %s, got %s", ef, cf)
-						}
-						if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
-							t.Errorf("clusterIP is not allocated: %v", clip)
-						}
-					}
+					proveClusterIPsAllocated(t, storage, nil, createdSvc)
 				})
 			}
 		})
@@ -5436,16 +5424,8 @@ func TestCreateDeleteReuse(t *testing.T) {
 			createdSvc := createdObj.(*api.Service)
 
 			// Ensure IPs and ports were allocated
-			for i, fam := range createdSvc.Spec.IPFamilies {
-				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-					t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
-				}
-			}
-			for _, p := range createdSvc.Spec.Ports {
-				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-					t.Errorf("expected port to be allocated: %v", p.NodePort)
-				}
-			}
+			proveClusterIPsAllocated(t, storage, tc.svc, createdSvc)
+			proveNodePortsAllocated(t, storage, tc.svc, createdSvc)
 
 			// Delete it
 			_, _, err = storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
@@ -5454,16 +5434,8 @@ func TestCreateDeleteReuse(t *testing.T) {
 			}
 
 			// Ensure IPs and ports were deallocated
-			for i, fam := range createdSvc.Spec.IPFamilies {
-				if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-					t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIPs[i])
-				}
-			}
-			for _, p := range createdSvc.Spec.Ports {
-				if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-					t.Errorf("expected port to not be allocated: %v", p.NodePort)
-				}
-			}
+			proveClusterIPsDeallocated(t, storage, createdSvc, nil)
+			proveNodePortsDeallocated(t, storage, createdSvc, nil)
 
 			// Force the same IPs and ports
 			svc2 := tc.svc.DeepCopy()
@@ -5479,17 +5451,8 @@ func TestCreateDeleteReuse(t *testing.T) {
 			}
 
 			// Ensure IPs and ports were allocated
-			for i, fam := range createdSvc.Spec.IPFamilies {
-				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-					t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
-				}
-			}
-			for _, p := range createdSvc.Spec.Ports {
-				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-					t.Errorf("expected port to be allocated: %v", p.NodePort)
-				}
-			}
-
+			proveClusterIPsAllocated(t, storage, svc2, createdSvc)
+			proveNodePortsAllocated(t, storage, svc2, createdSvc)
 		})
 	}
 }
@@ -6015,93 +5978,20 @@ func TestCreateDryRun(t *testing.T) {
 			}
 			createdSvc := createdObj.(*api.Service)
 
-			// Ensure IPs were allocated
+			// Ensure IPs were assigned
 			if netutils.ParseIPSloppy(createdSvc.Spec.ClusterIP) == nil {
 				t.Errorf("expected valid clusterIP: %q", createdSvc.Spec.ClusterIP)
 			}
-
-			// Ensure the IP allocators are clean.
-			if !tc.enableDualStack {
-				if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol], createdSvc.Spec.ClusterIP) {
-					t.Errorf("expected IP to not be allocated: %q", createdSvc.Spec.ClusterIP)
-				}
-			} else {
-				for _, ip := range createdSvc.Spec.ClusterIPs {
-					if netutils.ParseIPSloppy(ip) == nil {
-						t.Errorf("expected valid clusterIP: %q", createdSvc.Spec.ClusterIP)
-					}
-				}
-				for i, fam := range createdSvc.Spec.IPFamilies {
-					if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-						t.Errorf("expected IP to not be allocated: %q", createdSvc.Spec.ClusterIPs[i])
-					}
+			for _, ip := range createdSvc.Spec.ClusterIPs {
+				if netutils.ParseIPSloppy(ip) == nil {
+					t.Errorf("expected valid clusterIP: %q", createdSvc.Spec.ClusterIP)
 				}
 			}
 
+			// Ensure the allocators are clean.
+			proveClusterIPsDeallocated(t, storage, createdSvc, nil)
 			if tc.svc.Spec.Type != api.ServiceTypeClusterIP {
-				for _, p := range createdSvc.Spec.Ports {
-					if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-						t.Errorf("expected port to not be allocated: %d", p.NodePort)
-					}
-				}
-			}
-		})
-	}
-}
-
-func TestDeleteTypes(t *testing.T) {
-	testCases := []struct {
-		name string
-		svc  *api.Service
-	}{{
-		name: "type:ExternalName",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeExternalName),
-	}, {
-		name: "type:ClusterIP",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP),
-	}, {
-		name: "type:ClusterIP_headless",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetHeadless),
-	}, {
-		name: "type:NodePort",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeNodePort),
-	}, {
-		name: "type:LoadBalancer",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer),
-	}, {
-		name: "type:LoadBalancer_etp:Local",
-		svc: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-	}}
-
-	// This test is ONLY with the gate enabled.
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-
-	storage, _, server := newStorage(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
-	defer server.Terminate(t)
-	defer storage.Store.DestroyFunc()
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			ctx := genericapirequest.NewDefaultContext()
-			_, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("unexpected error creating service: %v", err)
-			}
-
-			_, deleted, err := storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
-			if err != nil {
-				t.Fatalf("unexpected error deleting service: %v", err)
-			}
-			if !deleted {
-				t.Fatalf("expected service to be deleted")
+				proveNodePortsDeallocated(t, storage, createdSvc, nil)
 			}
 		})
 	}
@@ -6141,19 +6031,9 @@ func TestDeleteWithFinalizer(t *testing.T) {
 	if !cmp.Equal(createdSvc, obj) {
 		t.Errorf("expected the result of Create() and Get() to match: %v", cmp.Diff(createdSvc, obj))
 	}
-	for i, fam := range createdSvc.Spec.IPFamilies {
-		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-			t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
-		}
-	}
-	for _, p := range createdSvc.Spec.Ports {
-		if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-			t.Errorf("expected port to be allocated: %v", p.NodePort)
-		}
-	}
-	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-		t.Errorf("expected port to be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
-	}
+	proveClusterIPsAllocated(t, storage, svc, createdSvc)
+	proveNodePortsAllocated(t, storage, svc, createdSvc)
+	proveHealthCheckNodePortAllocated(t, storage, svc, createdSvc)
 
 	// Try to delete it, but it should be blocked by the finalizer.
 	obj, deleted, err := storage.Delete(ctx, svcName, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
@@ -6170,19 +6050,9 @@ func TestDeleteWithFinalizer(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error getting service: %v", err)
 	}
-	for i, fam := range createdSvc.Spec.IPFamilies {
-		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-			t.Errorf("expected IP to be allocated: %v", createdSvc.Spec.ClusterIPs[i])
-		}
-	}
-	for _, p := range createdSvc.Spec.Ports {
-		if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-			t.Errorf("expected port to be allocated: %v", p.NodePort)
-		}
-	}
-	if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-		t.Errorf("expected port to be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
-	}
+	proveClusterIPsAllocated(t, storage, svc, createdSvc)
+	proveNodePortsAllocated(t, storage, svc, createdSvc)
+	proveHealthCheckNodePortAllocated(t, storage, svc, createdSvc)
 
 	// Clear the finalizer - should delete.
 	deletedSvc.Finalizers = nil
@@ -6198,19 +6068,9 @@ func TestDeleteWithFinalizer(t *testing.T) {
 	if err == nil {
 		t.Fatalf("unexpected success getting service")
 	}
-	for i, fam := range createdSvc.Spec.IPFamilies {
-		if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-			t.Errorf("expected IP to not be allocated: %v", createdSvc.Spec.ClusterIPs[i])
-		}
-	}
-	for _, p := range createdSvc.Spec.Ports {
-		if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-			t.Errorf("expected port to not be allocated: %v", p.NodePort)
-		}
-	}
-	if portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-		t.Errorf("expected port to not be allocated: %v", createdSvc.Spec.HealthCheckNodePort)
-	}
+	proveClusterIPsDeallocated(t, storage, createdSvc, nil)
+	proveNodePortsDeallocated(t, storage, createdSvc, nil)
+	proveHealthCheckNodePortDeallocated(t, storage, createdSvc, nil)
 }
 
 // Prove that a dry-run delete doesn't actually deallocate IPs or ports.
@@ -6254,19 +6114,9 @@ func TestDeleteDryRun(t *testing.T) {
 			createdSvc := createdObj.(*api.Service)
 
 			// Ensure IPs and ports were allocated
-			for i, fam := range createdSvc.Spec.IPFamilies {
-				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-					t.Errorf("expected IP to be allocated: %q", createdSvc.Spec.ClusterIPs[i])
-				}
-			}
-			for _, p := range createdSvc.Spec.Ports {
-				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-					t.Errorf("expected port to be allocated: %d", p.NodePort)
-				}
-			}
-			if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-				t.Errorf("expected port to be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
-			}
+			proveClusterIPsAllocated(t, storage, tc.svc, createdSvc)
+			proveNodePortsAllocated(t, storage, tc.svc, createdSvc)
+			proveHealthCheckNodePortAllocated(t, storage, tc.svc, createdSvc)
 
 			_, _, err = storage.Delete(ctx, tc.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}})
 			if err != nil {
@@ -6274,19 +6124,9 @@ func TestDeleteDryRun(t *testing.T) {
 			}
 
 			// Ensure they are still allocated.
-			for i, fam := range createdSvc.Spec.IPFamilies {
-				if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-					t.Errorf("expected IP to still be allocated: %q", createdSvc.Spec.ClusterIPs[i])
-				}
-			}
-			for _, p := range createdSvc.Spec.Ports {
-				if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-					t.Errorf("expected port to still be allocated: %d", p.NodePort)
-				}
-			}
-			if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-				t.Errorf("expected port to still be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
-			}
+			proveClusterIPsAllocated(t, storage, tc.svc, createdSvc)
+			proveNodePortsAllocated(t, storage, tc.svc, createdSvc)
+			proveHealthCheckNodePortAllocated(t, storage, tc.svc, createdSvc)
 		})
 	}
 }
@@ -6373,40 +6213,10 @@ func TestUpdateDryRun(t *testing.T) {
 			if tc.verifyDryAllocs {
 				// Dry allocs means no allocs on create.  Ensure values were
 				// NOT allocated.
-				for i, fam := range createdSvc.Spec.IPFamilies {
-					if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-						t.Errorf("expected IP to not be allocated: %q", createdSvc.Spec.ClusterIPs[i])
-					}
-				}
-				for _, p := range createdSvc.Spec.Ports {
-					if p.NodePort != 0 {
-						t.Errorf("expected port to not be assigned: %d", p.NodePort)
-						if portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-							t.Errorf("expected port to not be allocated: %d", p.NodePort)
-						}
-					}
-				}
-				if createdSvc.Spec.HealthCheckNodePort != 0 {
-					t.Errorf("expected HCNP to not be assigned: %d", createdSvc.Spec.HealthCheckNodePort)
-					if portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-						t.Errorf("expected HCNP to not be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
-					}
-				}
+				proveClusterIPsDeallocated(t, storage, nil, createdSvc)
 			} else {
 				// Ensure IPs were allocated
-				for i, fam := range createdSvc.Spec.IPFamilies {
-					if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[fam], createdSvc.Spec.ClusterIPs[i]) {
-						t.Errorf("expected IP to be allocated: %q", createdSvc.Spec.ClusterIPs[i])
-					}
-				}
-				for _, p := range createdSvc.Spec.Ports {
-					if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-						t.Errorf("expected port to be allocated: %d", p.NodePort)
-					}
-				}
-				if !portIsAllocated(t, storage.alloc.serviceNodePorts, createdSvc.Spec.HealthCheckNodePort) {
-					t.Errorf("expected port to be allocated: %d", createdSvc.Spec.HealthCheckNodePort)
-				}
+				proveClusterIPsAllocated(t, storage, nil, createdSvc)
 			}
 
 			// Update the object to the new state and check the results.
@@ -9795,6 +9605,7 @@ func TestUpdateIPsFromDualStack(t *testing.T) {
 	})
 }
 
+//FIXME: after all the tests are ported, move this all up near the top?
 type svcTestCase struct {
 	svc         *api.Service
 	expectError bool

From 12ac38f66192d2fadbc522b86b6eedb1318eff5f Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 21 Aug 2021 17:08:00 -0700
Subject: [PATCH 49/79] Svc REST: Beef up ports test, remove old form

---
 .../core/service/storage/rest_test.go         | 257 ---------------
 .../core/service/storage/storage_test.go      | 304 +++++++++++++++++-
 2 files changed, 291 insertions(+), 270 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index d03271af12c..832c042230c 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -18,7 +18,6 @@ package storage
 
 import (
 	"net"
-	"reflect"
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -119,262 +118,6 @@ func makeIPNet6(t *testing.T) *net.IPNet {
 	return net
 }
 
-func TestServiceRegistryUpdateUnspecifiedAllocations(t *testing.T) {
-	type proof func(t *testing.T, s *api.Service)
-	prove := func(proofs ...proof) []proof {
-		return proofs
-	}
-	proveClusterIP := func(idx int, ip string) proof {
-		return func(t *testing.T, s *api.Service) {
-			if want, got := ip, s.Spec.ClusterIPs[idx]; want != got {
-				t.Errorf("wrong ClusterIPs[%d]: want %q, got %q", idx, want, got)
-			}
-		}
-	}
-	proveNodePort := func(idx int, port int32) proof {
-		return func(t *testing.T, s *api.Service) {
-			got := s.Spec.Ports[idx].NodePort
-			if port > 0 && got != port {
-				t.Errorf("wrong Ports[%d].NodePort: want %d, got %d", idx, port, got)
-			} else if port < 0 && got == -port {
-				t.Errorf("wrong Ports[%d].NodePort: wanted anything but %d", idx, got)
-			}
-		}
-	}
-	proveHCNP := func(port int32) proof {
-		return func(t *testing.T, s *api.Service) {
-			got := s.Spec.HealthCheckNodePort
-			if port > 0 && got != port {
-				t.Errorf("wrong HealthCheckNodePort: want %d, got %d", port, got)
-			} else if port < 0 && got == -port {
-				t.Errorf("wrong HealthCheckNodePort: wanted anything but %d", got)
-			}
-		}
-	}
-
-	testCases := []struct {
-		name        string
-		create      *api.Service // Needs clusterIP, NodePort, and HealthCheckNodePort allocated
-		update      *api.Service // Needs clusterIP, NodePort, and/or HealthCheckNodePort blank
-		expectError bool
-		prove       []proof
-	}{{
-		name: "single-ip_single-port",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetClusterIPs("1.2.3.4"),
-			svctest.SetNodePorts(30093),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-		prove: prove(
-			proveClusterIP(0, "1.2.3.4"),
-			proveNodePort(0, 30093),
-			proveHCNP(30118)),
-	}, {
-		name: "multi-ip_multi-port",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-			svctest.SetClusterIPs("1.2.3.4", "2000::1"),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 30076),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP))),
-		prove: prove(
-			proveClusterIP(0, "1.2.3.4"),
-			proveClusterIP(1, "2000::1"),
-			proveNodePort(0, 30093),
-			proveNodePort(1, 30076),
-			proveHCNP(30118)),
-	}, {
-		name: "multi-ip_partial",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
-			svctest.SetClusterIPs("1.2.3.4", "2000::1"),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 30076),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetClusterIPs("1.2.3.4")),
-		expectError: true,
-	}, {
-		name: "multi-port_partial",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 30076),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 0)), // provide just 1 value
-		prove: prove(
-			proveNodePort(0, 30093),
-			proveNodePort(1, 30076),
-			proveHCNP(30118)),
-	}, {
-		name: "swap-ports",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 30076),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				// swapped from above
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP),
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP))),
-		prove: prove(
-			proveNodePort(0, 30076),
-			proveNodePort(1, 30093),
-			proveHCNP(30118)),
-	}, {
-		name: "partial-swap-ports",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 30076),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30076, 0), // set [0] to [1], omit [1]
-			svctest.SetHealthCheckNodePort(30118)),
-		prove: prove(
-			proveNodePort(0, 30076),
-			proveNodePort(1, -30076),
-			proveHCNP(30118)),
-	}, {
-		name: "swap-port-with-hcnp",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 30076),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30076, 30118)), // set [0] to [1], set [1] to HCNP
-		expectError: true,
-	}, {
-		name: "partial-swap-port-with-hcnp",
-		create: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30093, 30076),
-			svctest.SetHealthCheckNodePort(30118)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
-			svctest.SetNodePorts(30118, 0)), // set [0] to HCNP, omit [1]
-		expectError: true,
-	}}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			ctx := genericapirequest.NewDefaultContext()
-			storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
-			defer server.Terminate(t)
-
-			svc := tc.create.DeepCopy()
-			obj, err := storage.Create(ctx, svc.DeepCopy(), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if err != nil {
-				t.Fatalf("unexpected error on create: %v", err)
-			}
-			createdSvc := obj.(*api.Service)
-			if createdSvc.Spec.ClusterIP == "" {
-				t.Fatalf("expected ClusterIP to be set")
-			}
-			if len(createdSvc.Spec.ClusterIPs) == 0 {
-				t.Fatalf("expected ClusterIPs to be set")
-			}
-			for i := range createdSvc.Spec.Ports {
-				if createdSvc.Spec.Ports[i].NodePort == 0 {
-					t.Fatalf("expected NodePort[%d] to be set", i)
-				}
-			}
-			if createdSvc.Spec.HealthCheckNodePort == 0 {
-				t.Fatalf("expected HealthCheckNodePort to be set")
-			}
-
-			// Update - change the selector to be sure.
-			svc = tc.update.DeepCopy()
-			svc.Spec.Selector = map[string]string{"bar": "baz2"}
-			svc.ResourceVersion = createdSvc.ResourceVersion
-
-			obj, _, err = storage.Update(ctx, svc.Name, rest.DefaultUpdatedObjectInfo(svc.DeepCopy()),
-				rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-			if tc.expectError {
-				if err == nil {
-					t.Fatalf("unexpected success on update")
-				}
-				return
-			}
-			if err != nil {
-				t.Fatalf("unexpected error on update: %v", err)
-			}
-			updatedSvc := obj.(*api.Service)
-
-			if want, got := createdSvc.Spec.ClusterIP, updatedSvc.Spec.ClusterIP; want != got {
-				t.Errorf("expected ClusterIP to not change: wanted %v, got %v", want, got)
-			}
-			if want, got := createdSvc.Spec.ClusterIPs, updatedSvc.Spec.ClusterIPs; !reflect.DeepEqual(want, got) {
-				t.Errorf("expected ClusterIPs to not change: wanted %v, got %v", want, got)
-			}
-
-			for _, proof := range tc.prove {
-				proof(t, updatedSvc)
-			}
-		})
-	}
-}
-
 func TestServiceStorageValidatesUpdate(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()
 	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 5399b8169b7..d5fad0fd5dd 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -6296,6 +6296,272 @@ func TestUpdateDryRun(t *testing.T) {
 	}
 }
 
+func TestUpdatePatchAllocatedValues(t *testing.T) {
+	prove := func(proofs ...svcTestProof) []svcTestProof {
+		return proofs
+	}
+	proveClusterIP := func(idx int, ip string) svcTestProof {
+		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+			if want, got := ip, after.Spec.ClusterIPs[idx]; want != got {
+				t.Errorf("wrong ClusterIPs[%d]: want %q, got %q", idx, want, got)
+			}
+		}
+	}
+	proveNodePort := func(idx int, port int32) svcTestProof {
+		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+			got := after.Spec.Ports[idx].NodePort
+			if port > 0 && got != port {
+				t.Errorf("wrong Ports[%d].NodePort: want %d, got %d", idx, port, got)
+			} else if port < 0 && got == -port {
+				t.Errorf("wrong Ports[%d].NodePort: wanted anything but %d", idx, got)
+			}
+		}
+	}
+	proveHCNP := func(port int32) svcTestProof {
+		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+			got := after.Spec.HealthCheckNodePort
+			if port > 0 && got != port {
+				t.Errorf("wrong HealthCheckNodePort: want %d, got %d", port, got)
+			} else if port < 0 && got == -port {
+				t.Errorf("wrong HealthCheckNodePort: wanted anything but %d", got)
+			}
+		}
+	}
+
+	// each create needs clusterIP, NodePort, and HealthCheckNodePort allocated
+	// each update needs clusterIP, NodePort, and/or HealthCheckNodePort blank
+	testCases := []cudTestCase{{
+		name: "single-ip_single-port",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetClusterIPs("10.0.0.1"),
+				svctest.SetNodePorts(30093),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+			prove: prove(
+				proveClusterIP(0, "10.0.0.1"),
+				proveNodePort(0, 30093),
+				proveHCNP(30118)),
+		},
+	}, {
+		name: "multi-ip_multi-port",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 30076),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP))),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+			prove: prove(
+				proveClusterIP(0, "10.0.0.1"),
+				proveClusterIP(1, "2000::1"),
+				proveNodePort(0, 30093),
+				proveNodePort(1, 30076),
+				proveHCNP(30118)),
+		},
+	}, {
+		name: "multi-ip_partial",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetIPFamilyPolicy(api.IPFamilyPolicyPreferDualStack),
+				svctest.SetClusterIPs("10.0.0.1", "2000::1"),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 30076),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetClusterIPs("10.0.0.1")),
+			expectError: true,
+		},
+	}, {
+		name: "multi-port_partial",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 30076),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 0)), // provide just 1 value
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+			prove: prove(
+				proveNodePort(0, 30093),
+				proveNodePort(1, 30076),
+				proveHCNP(30118)),
+		},
+	}, {
+		name: "swap-ports",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 30076),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					// swapped from above
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP),
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP))),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+			prove: prove(
+				proveNodePort(0, 30076),
+				proveNodePort(1, 30093),
+				proveHCNP(30118)),
+		},
+	}, {
+		name: "partial-swap-ports",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 30076),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30076, 0), // set [0] to [1]'s value, omit [1]
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+			prove: prove(
+				proveNodePort(0, 30076),
+				proveNodePort(1, -30076),
+				proveHCNP(30118)),
+		},
+	}, {
+		name: "swap-port-with-hcnp",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 30076),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30076, 30118)), // set [0] to HCNP's value
+			expectError: true,
+		},
+	}, {
+		name: "partial-swap-port-with-hcnp",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30093, 30076),
+				svctest.SetHealthCheckNodePort(30118)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 867, intstr.FromInt(867), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 5309, intstr.FromInt(5309), api.ProtocolTCP)),
+				svctest.SetNodePorts(30118, 0)), // set [0] to HCNP's value, omit [1]
+			expectError: true,
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
 // Proves that updates from single-stack work.
 func TestUpdateIPsFromSingleStack(t *testing.T) {
 	prove := func(proofs ...svcTestProof) []svcTestProof {
@@ -10382,95 +10648,107 @@ func TestFeaturePorts(t *testing.T) {
 	}, {
 		name: "swap_ports",
 		create: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
 					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 		update: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP),
 					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 	}, {
 		name: "modify_ports",
 		create: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
 					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 		update: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 8080, intstr.FromInt(8080), api.ProtocolTCP),
 					svctest.MakeServicePort("q", 8443, intstr.FromInt(8443), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 	}, {
 		name: "modify_protos",
 		create: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
 					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 		update: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolUDP),
 					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolUDP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 	}, {
 		name: "modify_ports_and_protos",
 		create: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
 					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 		update: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("r", 53, intstr.FromInt(53), api.ProtocolTCP),
 					svctest.MakeServicePort("s", 53, intstr.FromInt(53), api.ProtocolUDP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 	}, {
 		name: "add_alt_proto",
 		create: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 		update: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 53, intstr.FromInt(53), api.ProtocolTCP),
 					svctest.MakeServicePort("q", 53, intstr.FromInt(53), api.ProtocolUDP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 	}, {
 		name: "wipe_all",
 		create: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts(
 					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
 					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP))),
 			expectClusterIPs: true,
+			expectNodePorts:  true,
 		},
 		update: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeClusterIP,
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
 				svctest.SetPorts()),
-			expectError: true,
+			expectError:     true,
+			expectNodePorts: true,
 		},
 	}}
 

From c71467def0b1c279c21ed8ceaed707eeb26deac8 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 21 Aug 2021 18:33:53 -0700
Subject: [PATCH 50/79] Svc REST: Remove overlapping rest_tests

Most are moved to storage_test
---
 .../core/service/storage/rest_test.go         | 106 ------------------
 1 file changed, 106 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index 832c042230c..e060a0094d6 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -22,7 +22,6 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
@@ -35,7 +34,6 @@ import (
 	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
 	netutils "k8s.io/utils/net"
-	utilpointer "k8s.io/utils/pointer"
 )
 
 func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *etcd3testing.EtcdTestServer) {
@@ -118,85 +116,6 @@ func makeIPNet6(t *testing.T) *net.IPNet {
 	return net
 }
 
-func TestServiceStorageValidatesUpdate(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	_, err := storage.Create(ctx, svctest.MakeService("foo"), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	failureCases := map[string]*api.Service{
-		"empty ID": svctest.MakeService(""),
-		"invalid selector": svctest.MakeService("", func(svc *api.Service) {
-			svc.Spec.Selector = map[string]string{"ThisSelectorFailsValidation": "ok"}
-		}),
-	}
-	for _, failureCase := range failureCases {
-		c, created, err := storage.Update(ctx, failureCase.Name, rest.DefaultUpdatedObjectInfo(failureCase), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-		if err == nil {
-			t.Errorf("expected error")
-		}
-		if c != nil || created {
-			t.Errorf("Expected nil object or created false")
-		}
-	}
-}
-
-func TestServiceRegistryUpdateLoadBalancerService(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	// Create non-loadbalancer.
-	svc1 := svctest.MakeService("foo")
-	obj, err := storage.Create(ctx, svc1, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-
-	// Modify to be loadbalancer.
-	svc2 := obj.(*api.Service).DeepCopy()
-	svc2.Spec.Type = api.ServiceTypeLoadBalancer
-	svc2.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeCluster
-	svc2.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true)
-	obj, _, err = storage.Update(ctx, svc2.Name, rest.DefaultUpdatedObjectInfo(svc2), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-
-	// Change port.
-	svc3 := obj.(*api.Service).DeepCopy()
-	svc3.Spec.Ports[0].Port = 6504
-	if _, _, err := storage.Update(ctx, svc3.Name, rest.DefaultUpdatedObjectInfo(svc3), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-}
-
-func TestServiceRegistryUpdateMultiPortLoadBalancerService(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-
-	// Create load balancer.
-	svc1 := svctest.MakeService("foo",
-		svctest.SetTypeLoadBalancer,
-		svctest.SetPorts(
-			svctest.MakeServicePort("p", 6502, intstr.FromInt(6502), api.ProtocolTCP),
-			svctest.MakeServicePort("q", 8086, intstr.FromInt(8086), api.ProtocolTCP)))
-	obj, err := storage.Create(ctx, svc1, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-
-	// Modify ports
-	svc2 := obj.(*api.Service).DeepCopy()
-	svc2.Spec.Ports[1].Port = 8088
-	if _, _, err := storage.Update(ctx, svc2.Name, rest.DefaultUpdatedObjectInfo(svc2), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-}
-
 // this is local because it's not fully fleshed out enough for general use.
 func makePod(name string, ips ...string) api.Pod {
 	p := api.Pod{
@@ -221,31 +140,6 @@ func makePod(name string, ips ...string) api.Pod {
 	return p
 }
 
-func TestServiceRegistryList(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	_, err := storage.Create(ctx, svctest.MakeService("foo"), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	_, err = storage.Create(ctx, svctest.MakeService("foo2"), rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	s, _ := storage.List(ctx, nil)
-	sl := s.(*api.ServiceList)
-	if len(sl.Items) != 2 {
-		t.Fatalf("Expected 2 services, but got %v", len(sl.Items))
-	}
-	if e, a := "foo", sl.Items[0].Name; e != a {
-		t.Errorf("Expected %v, but got %v", e, a)
-	}
-	if e, a := "foo2", sl.Items[1].Name; e != a {
-		t.Errorf("Expected %v, but got %v", e, a)
-	}
-}
-
 // Validate the internalTrafficPolicy field when set to "Cluster" then updated to "Local"
 func TestServiceRegistryInternalTrafficPolicyClusterThenLocal(t *testing.T) {
 	ctx := genericapirequest.NewDefaultContext()

From b6da6c9c0f25ce7aa34c3a528a15c55469430575 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 21 Aug 2021 18:39:49 -0700
Subject: [PATCH 51/79] Svc REST: Add InternalTrafficPolicy tests

Remove older form.
---
 pkg/api/service/testing/make.go               |   3 +-
 .../core/service/storage/rest_test.go         |  67 -----
 .../core/service/storage/storage_test.go      | 243 ++++++++++++++++--
 3 files changed, 218 insertions(+), 95 deletions(-)

diff --git a/pkg/api/service/testing/make.go b/pkg/api/service/testing/make.go
index ed0d4ddf347..87b86930cf4 100644
--- a/pkg/api/service/testing/make.go
+++ b/pkg/api/service/testing/make.go
@@ -48,7 +48,8 @@ func MakeService(name string, tweaks ...Tweak) *api.Service {
 	SetTypeClusterIP(svc)
 	// Default to 1 port
 	SetPorts(MakeServicePort("", 93, intstr.FromInt(76), api.ProtocolTCP))(svc)
-	// Default internalTrafficPolicy to "Cluster"
+	// Default internalTrafficPolicy to "Cluster".  This probably should not
+	// apply to ExternalName, but it went into beta and is not worth breaking.
 	SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyCluster)(svc)
 
 	for _, tweak := range tweaks {
diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
index e060a0094d6..9d8156ff04a 100644
--- a/pkg/registry/core/service/storage/rest_test.go
+++ b/pkg/registry/core/service/storage/rest_test.go
@@ -23,11 +23,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
-	"k8s.io/apiserver/pkg/registry/rest"
 	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
-	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	endpointstore "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
@@ -139,67 +136,3 @@ func makePod(name string, ips ...string) api.Pod {
 
 	return p
 }
-
-// Validate the internalTrafficPolicy field when set to "Cluster" then updated to "Local"
-func TestServiceRegistryInternalTrafficPolicyClusterThenLocal(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("internal-traffic-policy-cluster",
-		svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyCluster),
-	)
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if obj == nil || err != nil {
-		t.Errorf("Unexpected failure creating service %v", err)
-	}
-
-	createdSvc := obj.(*api.Service)
-	if *createdSvc.Spec.InternalTrafficPolicy != api.ServiceInternalTrafficPolicyCluster {
-		t.Errorf("Expecting internalTrafficPolicy field to have value Cluster, got: %s", *createdSvc.Spec.InternalTrafficPolicy)
-	}
-
-	update := createdSvc.DeepCopy()
-	local := api.ServiceInternalTrafficPolicyLocal
-	update.Spec.InternalTrafficPolicy = &local
-
-	updatedSvc, _, errUpdate := storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-	if errUpdate != nil {
-		t.Fatalf("unexpected error during update %v", errUpdate)
-	}
-	updatedService := updatedSvc.(*api.Service)
-	if *updatedService.Spec.InternalTrafficPolicy != api.ServiceInternalTrafficPolicyLocal {
-		t.Errorf("Expected internalTrafficPolicy to be Local, got: %s", *updatedService.Spec.InternalTrafficPolicy)
-	}
-}
-
-// Validate the internalTrafficPolicy field when set to "Local" and then updated to "Cluster"
-func TestServiceRegistryInternalTrafficPolicyLocalThenCluster(t *testing.T) {
-	ctx := genericapirequest.NewDefaultContext()
-	storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol})
-	defer server.Terminate(t)
-	svc := svctest.MakeService("internal-traffic-policy-cluster",
-		svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal),
-	)
-	obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-	if obj == nil || err != nil {
-		t.Errorf("Unexpected failure creating service %v", err)
-	}
-
-	createdSvc := obj.(*api.Service)
-	if *createdSvc.Spec.InternalTrafficPolicy != api.ServiceInternalTrafficPolicyLocal {
-		t.Errorf("Expecting internalTrafficPolicy field to have value Local, got: %s", *createdSvc.Spec.InternalTrafficPolicy)
-	}
-
-	update := createdSvc.DeepCopy()
-	cluster := api.ServiceInternalTrafficPolicyCluster
-	update.Spec.InternalTrafficPolicy = &cluster
-
-	updatedSvc, _, errUpdate := storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-	if errUpdate != nil {
-		t.Fatalf("unexpected error during update %v", errUpdate)
-	}
-	updatedService := updatedSvc.(*api.Service)
-	if *updatedService.Spec.InternalTrafficPolicy != api.ServiceInternalTrafficPolicyCluster {
-		t.Errorf("Expected internalTrafficPolicy to be Cluster, got: %s", *updatedService.Spec.InternalTrafficPolicy)
-	}
-}
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index d5fad0fd5dd..e26899a6c21 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -10121,40 +10121,45 @@ func helpTestCreateUpdateDeleteWithFamilies(t *testing.T, testCases []cudTestCas
 				return
 			}
 			verifyExpectations(t, storage, tc.create, tc.create.svc, createdSvc)
+			lastSvc := createdSvc
 
-			// Allow callers to do something between create and update.
-			if tc.beforeUpdate != nil {
-				tc.beforeUpdate(t, storage)
-			}
+			// The update phase is optional.
+			if tc.update.svc != nil {
+				// Allow callers to do something between create and update.
+				if tc.beforeUpdate != nil {
+					tc.beforeUpdate(t, storage)
+				}
 
-			// Update the object to the new state and check the results.
-			obj, created, err := storage.Update(ctx, tc.update.svc.Name,
-				rest.DefaultUpdatedObjectInfo(tc.update.svc), rest.ValidateAllObjectFunc,
-				rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-			if tc.update.expectError && err != nil {
-				return
+				// Update the object to the new state and check the results.
+				obj, created, err := storage.Update(ctx, tc.update.svc.Name,
+					rest.DefaultUpdatedObjectInfo(tc.update.svc), rest.ValidateAllObjectFunc,
+					rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
+				if tc.update.expectError && err != nil {
+					return
+				}
+				if err != nil {
+					t.Fatalf("unexpected error updating service: %v", err)
+				}
+				if tc.update.expectError && err == nil {
+					t.Fatalf("unexpected success updating service")
+				}
+				if created {
+					t.Fatalf("unexpected create-on-update")
+				}
+				updatedSvc := obj.(*api.Service)
+				if !verifyEquiv(t, "update", &tc.update, updatedSvc) {
+					return
+				}
+				verifyExpectations(t, storage, tc.update, createdSvc, updatedSvc)
+				lastSvc = updatedSvc
 			}
-			if err != nil {
-				t.Fatalf("unexpected error updating service: %v", err)
-			}
-			if tc.update.expectError && err == nil {
-				t.Fatalf("unexpected success updating service")
-			}
-			if created {
-				t.Fatalf("unexpected create-on-update")
-			}
-			updatedSvc := obj.(*api.Service)
-			if !verifyEquiv(t, "update", &tc.update, updatedSvc) {
-				return
-			}
-			verifyExpectations(t, storage, tc.update, createdSvc, updatedSvc)
 
 			// Delete the object and check the results.
-			_, _, err = storage.Delete(ctx, tc.update.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			_, _, err = storage.Delete(ctx, tc.create.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
 			if err != nil {
 				t.Fatalf("unexpected error deleting service: %v", err)
 			}
-			verifyExpectations(t, storage, svcTestCase{ /* all false */ }, updatedSvc, nil)
+			verifyExpectations(t, storage, svcTestCase{ /* all false */ }, lastSvc, nil)
 		})
 	}
 }
@@ -11331,8 +11336,192 @@ func TestFeatureExternalTrafficPolicy(t *testing.T) {
 	helpTestCreateUpdateDelete(t, testCases)
 }
 
+func TestFeatureInternalTrafficPolicy(t *testing.T) {
+	prove := func(proofs ...svcTestProof) []svcTestProof {
+		return proofs
+	}
+	proveITP := func(want api.ServiceInternalTrafficPolicyType) svcTestProof {
+		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+			t.Helper()
+			if got := after.Spec.InternalTrafficPolicy; got == nil {
+				if want != "" {
+					t.Errorf("internalTrafficPolicy was nil")
+				}
+			} else if *got != want {
+				if want == "" {
+					want = "nil"
+				}
+				t.Errorf("wrong internalTrafficPoilcy: expected %s, got %s", want, *got)
+			}
+		}
+	}
+
+	testCases := []cudTestCase{{
+		name: "ExternalName_policy:none-ExternalName_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName),
+			prove: prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			prove: prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "ExternalName_policy:Cluster-ExternalName_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyCluster)),
+			prove: prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeExternalName,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			prove: prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "ClusterIP_policy:none-ClusterIP_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectClusterIPs: true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "ClusterIP_policy:Cluster-ClusterIP_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyCluster)),
+			expectClusterIPs: true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeClusterIP,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectClusterIPs: true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "NodePort_policy:none-NodePort_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "NodePort_policy:Cluster-NodePort_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyCluster)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeNodePort,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "LoadBalancer_policy:none-LoadBalancer_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "LoadBalancer_policy:Cluster-LoadBalancer_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyCluster)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetTypeLoadBalancer,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+			prove:            prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "Headless_policy:none-Headless_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless),
+			expectHeadless: true,
+			prove:          prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectHeadless: true,
+			prove:          prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}, {
+		name: "Headless_policy:Cluster-Headless_policy:Local",
+		create: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyCluster)),
+			expectHeadless: true,
+			prove:          prove(proveITP(api.ServiceInternalTrafficPolicyCluster)),
+		},
+		update: svcTestCase{
+			svc: svctest.MakeService("foo",
+				svctest.SetHeadless,
+				svctest.SetInternalTrafficPolicy(api.ServiceInternalTrafficPolicyLocal)),
+			expectHeadless: true,
+			prove:          prove(proveITP(api.ServiceInternalTrafficPolicyLocal)),
+		},
+	}}
+
+	helpTestCreateUpdateDelete(t, testCases)
+}
+
 // FIXME: externalIPs, lbip,
-// lbsourceranges, externalname, itp, PublishNotReadyAddresses,
+// lbsourceranges, externalname, PublishNotReadyAddresses,
 // ipfamilypolicy and list,
 // AllocateLoadBalancerNodePorts, LoadBalancerClass, status
 

From 4ac7c73b2ee6540939e48e88453e15f2e1e5c626 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sat, 21 Aug 2021 20:13:51 -0700
Subject: [PATCH 52/79] Svc REST: Remove old rest_test

All the tests have been ported to storage_test.go
---
 .../core/service/storage/rest_test.go         | 138 ------------------
 .../core/service/storage/storage_test.go      |  24 +++
 2 files changed, 24 insertions(+), 138 deletions(-)
 delete mode 100644 pkg/registry/core/service/storage/rest_test.go

diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go
deleted file mode 100644
index 9d8156ff04a..00000000000
--- a/pkg/registry/core/service/storage/rest_test.go
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package storage
-
-import (
-	"net"
-	"testing"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	utilnet "k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/apiserver/pkg/registry/generic"
-	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
-	api "k8s.io/kubernetes/pkg/apis/core"
-	endpointstore "k8s.io/kubernetes/pkg/registry/core/endpoint/storage"
-	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
-	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
-	"k8s.io/kubernetes/pkg/registry/registrytest"
-	netutils "k8s.io/utils/net"
-)
-
-func NewTestREST(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *etcd3testing.EtcdTestServer) {
-	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
-
-	var rPrimary ipallocator.Interface
-	var rSecondary ipallocator.Interface
-
-	if len(ipFamilies) < 1 || len(ipFamilies) > 2 {
-		t.Fatalf("unexpected ipfamilies passed: %v", ipFamilies)
-	}
-	for i, family := range ipFamilies {
-		var r ipallocator.Interface
-		var err error
-		switch family {
-		case api.IPv4Protocol:
-			r, err = ipallocator.NewInMemory(makeIPNet(t))
-			if err != nil {
-				t.Fatalf("cannot create CIDR Range %v", err)
-			}
-		case api.IPv6Protocol:
-			r, err = ipallocator.NewInMemory(makeIPNet6(t))
-			if err != nil {
-				t.Fatalf("cannot create CIDR Range %v", err)
-			}
-		}
-		switch i {
-		case 0:
-			rPrimary = r
-		case 1:
-			rSecondary = r
-		}
-	}
-
-	portRange := utilnet.PortRange{Base: 30000, Size: 1000}
-	portAllocator, err := portallocator.NewInMemory(portRange)
-	if err != nil {
-		t.Fatalf("cannot create port allocator %v", err)
-	}
-
-	ipAllocators := map[api.IPFamily]ipallocator.Interface{
-		rPrimary.IPFamily(): rPrimary,
-	}
-	if rSecondary != nil {
-		ipAllocators[rSecondary.IPFamily()] = rSecondary
-	}
-
-	restOptions := generic.RESTOptions{
-		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
-		Decorator:               generic.UndecoratedStorage,
-		DeleteCollectionWorkers: 1,
-		ResourcePrefix:          "services",
-	}
-	endpoints, err := endpointstore.NewREST(generic.RESTOptions{
-		StorageConfig:  etcdStorage,
-		Decorator:      generic.UndecoratedStorage,
-		ResourcePrefix: "endpoints",
-	})
-
-	rest, _, _, err := NewREST(restOptions, api.IPv4Protocol, ipAllocators, portAllocator, endpoints, nil, nil)
-	if err != nil {
-		t.Fatalf("unexpected error from REST storage: %v", err)
-	}
-
-	return rest, server
-}
-
-func makeIPNet(t *testing.T) *net.IPNet {
-	_, net, err := netutils.ParseCIDRSloppy("1.2.3.0/24")
-	if err != nil {
-		t.Error(err)
-	}
-	return net
-}
-func makeIPNet6(t *testing.T) *net.IPNet {
-	_, net, err := netutils.ParseCIDRSloppy("2000::/108")
-	if err != nil {
-		t.Error(err)
-	}
-	return net
-}
-
-// this is local because it's not fully fleshed out enough for general use.
-func makePod(name string, ips ...string) api.Pod {
-	p := api.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: metav1.NamespaceDefault,
-		},
-		Spec: api.PodSpec{
-			RestartPolicy: api.RestartPolicyAlways,
-			DNSPolicy:     api.DNSDefault,
-			Containers:    []api.Container{{Name: "ctr", Image: "img", ImagePullPolicy: api.PullIfNotPresent, TerminationMessagePolicy: api.TerminationMessageReadFile}},
-		},
-		Status: api.PodStatus{
-			PodIPs: []api.PodIP{},
-		},
-	}
-
-	for _, ip := range ips {
-		p.Status.PodIPs = append(p.Status.PodIPs, api.PodIP{IP: ip})
-	}
-
-	return p
-}
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index e26899a6c21..5d72cd610ed 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -11525,6 +11525,30 @@ func TestFeatureInternalTrafficPolicy(t *testing.T) {
 // ipfamilypolicy and list,
 // AllocateLoadBalancerNodePorts, LoadBalancerClass, status
 
+// this is local because it's not fully fleshed out enough for general use.
+func makePod(name string, ips ...string) api.Pod {
+	p := api.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: api.PodSpec{
+			RestartPolicy: api.RestartPolicyAlways,
+			DNSPolicy:     api.DNSDefault,
+			Containers:    []api.Container{{Name: "ctr", Image: "img", ImagePullPolicy: api.PullIfNotPresent, TerminationMessagePolicy: api.TerminationMessageReadFile}},
+		},
+		Status: api.PodStatus{
+			PodIPs: []api.PodIP{},
+		},
+	}
+
+	for _, ip := range ips {
+		p.Status.PodIPs = append(p.Status.PodIPs, api.PodIP{IP: ip})
+	}
+
+	return p
+}
+
 func TestServiceRegistryResourceLocation(t *testing.T) {
 	pods := []api.Pod{
 		makePod("unnamed", "1.2.3.4", "1.2.3.5"),

From 4718a0f2147ee53787b2c92ad48f2d9f6ded7489 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Sun, 22 Aug 2021 12:30:47 -0700
Subject: [PATCH 53/79] DeepCopy() input objects in Service REST test

Since the PR to do this deeper in the stack was declined, we'll do it
ourselves.  This ensures that we don't accidentally mutate the input and
then compare that mutated form to the result (which caused previous test
failures).
---
 .../core/service/storage/storage_test.go      | 57 ++++++++++++-------
 1 file changed, 36 insertions(+), 21 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 5d72cd610ed..bb06ea575aa 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package storage
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"reflect"
@@ -87,11 +88,11 @@ func portIsAllocated(t *testing.T, alloc portallocator.Interface, port int32) bo
 	return alloc.Has(int(port))
 }
 
-func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
+func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*wrapperRESTForTests, *StatusREST, *etcd3testing.EtcdTestServer) {
 	return newStorageWithPods(t, ipFamilies, nil, nil)
 }
 
-func newStorageWithPods(t *testing.T, ipFamilies []api.IPFamily, pods []api.Pod, endpoints []*api.Endpoints) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
+func newStorageWithPods(t *testing.T, ipFamilies []api.IPFamily, pods []api.Pod, endpoints []*api.Endpoints) (*wrapperRESTForTests, *StatusREST, *etcd3testing.EtcdTestServer) {
 	etcdStorage, server := registrytest.NewEtcdStorage(t, "")
 	restOptions := generic.RESTOptions{
 		StorageConfig:           etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
@@ -158,7 +159,21 @@ func newStorageWithPods(t *testing.T, ipFamilies []api.IPFamily, pods []api.Pod,
 	if err != nil {
 		t.Fatalf("unexpected error from REST storage: %v", err)
 	}
-	return serviceStorage, statusStorage, server
+	return &wrapperRESTForTests{serviceStorage}, statusStorage, server
+}
+
+// wrapperRESTForTests is a *trivial* wrapper for the real REST, which allows us to do
+// things that are specifically to enhance test safety.
+type wrapperRESTForTests struct {
+	*GenericREST
+}
+
+func (f *wrapperRESTForTests) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
+	// Making a DeepCopy here ensures that any in-place mutations of the input
+	// are not going to propagate to verification code, which used to happen
+	// resulting in tests that passed when they shouldn't have.
+	obj = obj.DeepCopyObject()
+	return f.GenericREST.Create(ctx, obj, createValidation, options)
 }
 
 // This is used in generic registry tests.
@@ -6301,14 +6316,14 @@ func TestUpdatePatchAllocatedValues(t *testing.T) {
 		return proofs
 	}
 	proveClusterIP := func(idx int, ip string) svcTestProof {
-		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+		return func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 			if want, got := ip, after.Spec.ClusterIPs[idx]; want != got {
 				t.Errorf("wrong ClusterIPs[%d]: want %q, got %q", idx, want, got)
 			}
 		}
 	}
 	proveNodePort := func(idx int, port int32) svcTestProof {
-		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+		return func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 			got := after.Spec.Ports[idx].NodePort
 			if port > 0 && got != port {
 				t.Errorf("wrong Ports[%d].NodePort: want %d, got %d", idx, port, got)
@@ -6318,7 +6333,7 @@ func TestUpdatePatchAllocatedValues(t *testing.T) {
 		}
 	}
 	proveHCNP := func(port int32) svcTestProof {
-		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+		return func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 			got := after.Spec.HealthCheckNodePort
 			if port > 0 && got != port {
 				t.Errorf("wrong HealthCheckNodePort: want %d, got %d", port, got)
@@ -6568,7 +6583,7 @@ func TestUpdateIPsFromSingleStack(t *testing.T) {
 		return proofs
 	}
 	proveNumFamilies := func(n int) svcTestProof {
-		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+		return func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 			t.Helper()
 			if got := len(after.Spec.IPFamilies); got != n {
 				t.Errorf("wrong number of ipFamilies: expected %d, got %d", n, got)
@@ -7328,7 +7343,7 @@ func TestUpdateIPsFromSingleStack(t *testing.T) {
 			expectClusterIPs: true,
 			prove:            prove(proveNumFamilies(1)),
 		},
-		beforeUpdate: func(t *testing.T, storage *GenericREST) {
+		beforeUpdate: func(t *testing.T, storage *wrapperRESTForTests) {
 			alloc := storage.alloc.serviceIPAllocatorsByFamily[api.IPv6Protocol]
 			ip := "2000::1"
 			if err := alloc.Allocate(netutils.ParseIPSloppy(ip)); err != nil {
@@ -8036,7 +8051,7 @@ func TestUpdateIPsFromSingleStack(t *testing.T) {
 			expectClusterIPs: true,
 			prove:            prove(proveNumFamilies(1)),
 		},
-		beforeUpdate: func(t *testing.T, storage *GenericREST) {
+		beforeUpdate: func(t *testing.T, storage *wrapperRESTForTests) {
 			alloc := storage.alloc.serviceIPAllocatorsByFamily[api.IPv4Protocol]
 			ip := "10.0.0.1"
 			if err := alloc.Allocate(netutils.ParseIPSloppy(ip)); err != nil {
@@ -8280,7 +8295,7 @@ func TestUpdateIPsFromDualStack(t *testing.T) {
 		return proofs
 	}
 	proveNumFamilies := func(n int) svcTestProof {
-		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+		return func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 			t.Helper()
 			if got := len(after.Spec.IPFamilies); got != n {
 				t.Errorf("wrong number of ipFamilies: expected %d, got %d", n, got)
@@ -9889,7 +9904,7 @@ type svcTestCase struct {
 	prove []svcTestProof
 }
 
-type svcTestProof func(t *testing.T, storage *GenericREST, before, after *api.Service)
+type svcTestProof func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service)
 
 func callName(before, after *api.Service) string {
 	if before == nil && after != nil {
@@ -9904,7 +9919,7 @@ func callName(before, after *api.Service) string {
 	panic("this test is broken: before and after are both nil")
 }
 
-func proveClusterIPsAllocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+func proveClusterIPsAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 	t.Helper()
 
 	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
@@ -9980,7 +9995,7 @@ func proveClusterIPsAllocated(t *testing.T, storage *GenericREST, before, after
 	}
 }
 
-func proveClusterIPsDeallocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+func proveClusterIPsDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 	t.Helper()
 
 	if after != nil && after.Spec.ClusterIP != api.ClusterIPNone {
@@ -10007,7 +10022,7 @@ func proveClusterIPsDeallocated(t *testing.T, storage *GenericREST, before, afte
 	}
 }
 
-func proveHeadless(t *testing.T, storage *GenericREST, before, after *api.Service) {
+func proveHeadless(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 	t.Helper()
 
 	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
@@ -10018,7 +10033,7 @@ func proveHeadless(t *testing.T, storage *GenericREST, before, after *api.Servic
 	}
 }
 
-func proveNodePortsAllocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+func proveNodePortsAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 	t.Helper()
 
 	for _, p := range after.Spec.Ports {
@@ -10028,7 +10043,7 @@ func proveNodePortsAllocated(t *testing.T, storage *GenericREST, before, after *
 	}
 }
 
-func proveNodePortsDeallocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+func proveNodePortsDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 	t.Helper()
 
 	if after != nil {
@@ -10048,7 +10063,7 @@ func proveNodePortsDeallocated(t *testing.T, storage *GenericREST, before, after
 	}
 }
 
-func proveHealthCheckNodePortAllocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+func proveHealthCheckNodePortAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 	t.Helper()
 
 	if !portIsAllocated(t, storage.alloc.serviceNodePorts, after.Spec.HealthCheckNodePort) {
@@ -10056,7 +10071,7 @@ func proveHealthCheckNodePortAllocated(t *testing.T, storage *GenericREST, befor
 	}
 }
 
-func proveHealthCheckNodePortDeallocated(t *testing.T, storage *GenericREST, before, after *api.Service) {
+func proveHealthCheckNodePortDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 	t.Helper()
 
 	if after != nil {
@@ -10076,7 +10091,7 @@ type cudTestCase struct {
 	name         string
 	line         string // if not empty, will be logged with errors, use line() to set
 	create       svcTestCase
-	beforeUpdate func(t *testing.T, storage *GenericREST)
+	beforeUpdate func(t *testing.T, storage *wrapperRESTForTests)
 	update       svcTestCase
 }
 
@@ -10380,7 +10395,7 @@ func TestVerifyEquiv(t *testing.T) {
 	}
 }
 
-func verifyExpectations(t *testing.T, storage *GenericREST, tc svcTestCase, before, after *api.Service) {
+func verifyExpectations(t *testing.T, storage *wrapperRESTForTests, tc svcTestCase, before, after *api.Service) {
 	t.Helper()
 
 	if tc.expectClusterIPs {
@@ -11341,7 +11356,7 @@ func TestFeatureInternalTrafficPolicy(t *testing.T) {
 		return proofs
 	}
 	proveITP := func(want api.ServiceInternalTrafficPolicyType) svcTestProof {
-		return func(t *testing.T, storage *GenericREST, before, after *api.Service) {
+		return func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
 			t.Helper()
 			if got := after.Spec.InternalTrafficPolicy; got == nil {
 				if want != "" {

From 4ff4160e34bfb26816e2d997f7e95d9d92e44833 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Mon, 23 Aug 2021 19:36:02 -0700
Subject: [PATCH 54/79] Svc REST: Move normalizeClusterIPs to storage pkg

All the meaningful callers of it are in that pkg anyway.  Removes some
FIXMEs.
---
 pkg/registry/core/service/storage/storage.go  |  69 +++++++-
 .../core/service/storage/storage_test.go      | 157 +++++++++++++++++
 pkg/registry/core/service/strategy.go         |  68 --------
 pkg/registry/core/service/strategy_test.go    | 158 ------------------
 4 files changed, 223 insertions(+), 229 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 70f718b0dcc..7ba08a78646 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -222,7 +222,7 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 	// We might find Services that were written before ClusterIP became plural.
 	// We still want to present a consistent view of them.
 	// NOTE: the args are (old, new)
-	svcreg.NormalizeClusterIPs(nil, service)
+	normalizeClusterIPs(nil, service)
 
 	// The rest of this does not apply unless dual-stack is enabled.
 	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
@@ -324,7 +324,7 @@ func (r *GenericREST) beginCreate(ctx context.Context, obj runtime.Object, optio
 	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
 	// early, before anyone looks at them.
 	// NOTE: the args are (old, new)
-	svcreg.NormalizeClusterIPs(nil, svc)
+	normalizeClusterIPs(nil, svc)
 
 	// Allocate IPs and ports. If we had a transactional store, this would just
 	// be part of the larger transaction.  We don't have that, so we have to do
@@ -359,7 +359,7 @@ func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Objec
 	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
 	// early, before anyone looks at them.
 	// NOTE: the args are (old, new)
-	svcreg.NormalizeClusterIPs(oldSvc, newSvc)
+	normalizeClusterIPs(oldSvc, newSvc)
 
 	// Allocate and initialize fields.
 	txn, err := r.alloc.allocateUpdate(newSvc, oldSvc, dryrun.IsDryRun(options.DryRun))
@@ -451,3 +451,66 @@ func (r *GenericREST) ResourceLocation(ctx context.Context, id string) (*url.URL
 	}
 	return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", id))
 }
+
+// normalizeClusterIPs adjust clusterIPs based on ClusterIP.  This must not
+// consider any other fields.
+func normalizeClusterIPs(oldSvc, newSvc *api.Service) {
+	// In all cases here, we don't need to over-think the inputs.  Validation
+	// will be called on the new object soon enough.  All this needs to do is
+	// try to divine what user meant with these linked fields. The below
+	// is verbosely written for clarity.
+
+	// **** IMPORTANT *****
+	// as a governing rule. User must (either)
+	// -- Use singular only (old client)
+	// -- singular and plural fields (new clients)
+
+	if oldSvc == nil {
+		// This was a create operation.
+		// User specified singular and not plural (e.g. an old client), so init
+		// plural for them.
+		if len(newSvc.Spec.ClusterIP) > 0 && len(newSvc.Spec.ClusterIPs) == 0 {
+			newSvc.Spec.ClusterIPs = []string{newSvc.Spec.ClusterIP}
+			return
+		}
+
+		// we don't init singular based on plural because
+		// new client must use both fields
+
+		// Either both were not specified (will be allocated) or both were
+		// specified (will be validated).
+		return
+	}
+
+	// This was an update operation
+
+	// ClusterIPs were cleared by an old client which was trying to patch
+	// some field and didn't provide ClusterIPs
+	if len(oldSvc.Spec.ClusterIPs) > 0 && len(newSvc.Spec.ClusterIPs) == 0 {
+		// if ClusterIP is the same, then it is an old client trying to
+		// patch service and didn't provide ClusterIPs
+		if oldSvc.Spec.ClusterIP == newSvc.Spec.ClusterIP {
+			newSvc.Spec.ClusterIPs = oldSvc.Spec.ClusterIPs
+		}
+	}
+
+	// clusterIP is not the same
+	if oldSvc.Spec.ClusterIP != newSvc.Spec.ClusterIP {
+		// this is a client trying to clear it
+		if len(oldSvc.Spec.ClusterIP) > 0 && len(newSvc.Spec.ClusterIP) == 0 {
+			// if clusterIPs are the same, then clear on their behalf
+			if sameClusterIPs(oldSvc, newSvc) {
+				newSvc.Spec.ClusterIPs = nil
+			}
+
+			// if they provided nil, then we are fine (handled by patching case above)
+			// if they changed it then validation will catch it
+		} else {
+			// ClusterIP has changed but not cleared *and* ClusterIPs are the same
+			// then we set ClusterIPs based on ClusterIP
+			if sameClusterIPs(oldSvc, newSvc) {
+				newSvc.Spec.ClusterIPs = []string{newSvc.Spec.ClusterIP}
+			}
+		}
+	}
+}
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index bb06ea575aa..652cb9db10f 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -296,6 +296,163 @@ func TestGenericCategories(t *testing.T) {
 	registrytest.AssertCategories(t, storage, expected)
 }
 
+func TestNormalizeClusterIPs(t *testing.T) {
+	makeServiceWithClusterIp := func(clusterIP string, clusterIPs []string) *api.Service {
+		return &api.Service{
+			Spec: api.ServiceSpec{
+				ClusterIP:  clusterIP,
+				ClusterIPs: clusterIPs,
+			},
+		}
+	}
+
+	testCases := []struct {
+		name               string
+		oldService         *api.Service
+		newService         *api.Service
+		expectedClusterIP  string
+		expectedClusterIPs []string
+	}{
+		{
+			name:               "new - only clusterip used",
+			oldService:         nil,
+			newService:         makeServiceWithClusterIp("10.0.0.10", nil),
+			expectedClusterIP:  "10.0.0.10",
+			expectedClusterIPs: []string{"10.0.0.10"},
+		},
+		{
+			name:               "new - only clusterips used",
+			oldService:         nil,
+			newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
+			expectedClusterIP:  "", // this is a validation issue, and validation will catch it
+			expectedClusterIPs: []string{"10.0.0.10"},
+		},
+		{
+			name:               "new - both used",
+			oldService:         nil,
+			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			expectedClusterIP:  "10.0.0.10",
+			expectedClusterIPs: []string{"10.0.0.10"},
+		},
+		{
+			name:               "update - no change",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			expectedClusterIP:  "10.0.0.10",
+			expectedClusterIPs: []string{"10.0.0.10"},
+		},
+		{
+			name:               "update - malformed change",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11"}),
+			expectedClusterIP:  "10.0.0.11",
+			expectedClusterIPs: []string{"10.0.0.11"},
+		},
+		{
+			name:               "update - malformed change on secondary ip",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+			newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11", "3000::1"}),
+			expectedClusterIP:  "10.0.0.11",
+			expectedClusterIPs: []string{"10.0.0.11", "3000::1"},
+		},
+		{
+			name:               "update - upgrade",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+			expectedClusterIP:  "10.0.0.10",
+			expectedClusterIPs: []string{"10.0.0.10", "2000::1"},
+		},
+		{
+			name:               "update - downgrade",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			expectedClusterIP:  "10.0.0.10",
+			expectedClusterIPs: []string{"10.0.0.10"},
+		},
+		{
+			name:               "update - user cleared cluster IP",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
+			expectedClusterIP:  "",
+			expectedClusterIPs: nil,
+		},
+		{
+			name:               "update - user cleared clusterIPs", // *MUST* REMAIN FOR OLD CLIENTS
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("10.0.0.10", nil),
+			expectedClusterIP:  "10.0.0.10",
+			expectedClusterIPs: []string{"10.0.0.10"},
+		},
+		{
+			name:               "update - user cleared both",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("", nil),
+			expectedClusterIP:  "",
+			expectedClusterIPs: nil,
+		},
+		{
+			name:               "update - user cleared ClusterIP but changed clusterIPs",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("", []string{"10.0.0.11"}),
+			expectedClusterIP:  "", /* validation catches this */
+			expectedClusterIPs: []string{"10.0.0.11"},
+		},
+		{
+			name:               "update - user cleared ClusterIPs but changed ClusterIP",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+			newService:         makeServiceWithClusterIp("10.0.0.11", nil),
+			expectedClusterIP:  "10.0.0.11",
+			expectedClusterIPs: nil,
+		},
+		{
+			name:               "update - user changed from None to ClusterIP",
+			oldService:         makeServiceWithClusterIp("None", []string{"None"}),
+			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"None"}),
+			expectedClusterIP:  "10.0.0.10",
+			expectedClusterIPs: []string{"10.0.0.10"},
+		},
+		{
+			name:               "update - user changed from ClusterIP to None",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+			newService:         makeServiceWithClusterIp("None", []string{"10.0.0.10"}),
+			expectedClusterIP:  "None",
+			expectedClusterIPs: []string{"None"},
+		},
+		{
+			name:               "update - user changed from ClusterIP to None and changed ClusterIPs in a dual stack (new client making a mistake)",
+			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+			newService:         makeServiceWithClusterIp("None", []string{"10.0.0.11", "2000::1"}),
+			expectedClusterIP:  "None",
+			expectedClusterIPs: []string{"10.0.0.11", "2000::1"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			normalizeClusterIPs(tc.oldService, tc.newService)
+
+			if tc.newService == nil {
+				t.Fatalf("unexpected new service to be nil")
+			}
+
+			if tc.newService.Spec.ClusterIP != tc.expectedClusterIP {
+				t.Fatalf("expected clusterIP [%v] got [%v]", tc.expectedClusterIP, tc.newService.Spec.ClusterIP)
+			}
+
+			if len(tc.newService.Spec.ClusterIPs) != len(tc.expectedClusterIPs) {
+				t.Fatalf("expected  clusterIPs %v got %v", tc.expectedClusterIPs, tc.newService.Spec.ClusterIPs)
+			}
+
+			for idx, clusterIP := range tc.newService.Spec.ClusterIPs {
+				if clusterIP != tc.expectedClusterIPs[idx] {
+					t.Fatalf("expected clusterIP [%v] at index[%v] got [%v]", tc.expectedClusterIPs[idx], idx, tc.newService.Spec.ClusterIPs[idx])
+
+				}
+			}
+		})
+	}
+}
+
 func TestServiceDefaultOnRead(t *testing.T) {
 	// Helper makes a mostly-valid ServiceList.  Test-cases can tweak it as needed.
 	makeServiceList := func(tweaks ...svctest.Tweak) *api.ServiceList {
diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go
index 346cf2db6eb..b4bd7fa45fd 100644
--- a/pkg/registry/core/service/strategy.go
+++ b/pkg/registry/core/service/strategy.go
@@ -109,8 +109,6 @@ func (strategy svcStrategy) PrepareForCreate(ctx context.Context, obj runtime.Ob
 	service := obj.(*api.Service)
 	service.Status = api.ServiceStatus{}
 
-	//FIXME: Normalize is now called from BeginCreate in pkg/registry/core/service/storage
-	NormalizeClusterIPs(nil, service)
 	dropServiceDisabledFields(service, nil)
 }
 
@@ -120,8 +118,6 @@ func (strategy svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runti
 	oldService := old.(*api.Service)
 	newService.Status = oldService.Status
 
-	//FIXME: Normalize is now called from BeginUpdate in pkg/registry/core/service/storage
-	NormalizeClusterIPs(oldService, newService)
 	dropServiceDisabledFields(newService, oldService)
 	dropTypeDependentFields(newService, oldService)
 }
@@ -361,70 +357,6 @@ func PatchAllocatedValues(newSvc, oldSvc *api.Service) {
 	}
 }
 
-// NormalizeClusterIPs adjust clusterIPs based on ClusterIP.  This must not
-// consider any other fields.
-//FIXME: move this to pkg/registry/core/service/storage
-func NormalizeClusterIPs(oldSvc, newSvc *api.Service) {
-	// In all cases here, we don't need to over-think the inputs.  Validation
-	// will be called on the new object soon enough.  All this needs to do is
-	// try to divine what user meant with these linked fields. The below
-	// is verbosely written for clarity.
-
-	// **** IMPORTANT *****
-	// as a governing rule. User must (either)
-	// -- Use singular only (old client)
-	// -- singular and plural fields (new clients)
-
-	if oldSvc == nil {
-		// This was a create operation.
-		// User specified singular and not plural (e.g. an old client), so init
-		// plural for them.
-		if len(newSvc.Spec.ClusterIP) > 0 && len(newSvc.Spec.ClusterIPs) == 0 {
-			newSvc.Spec.ClusterIPs = []string{newSvc.Spec.ClusterIP}
-			return
-		}
-
-		// we don't init singular based on plural because
-		// new client must use both fields
-
-		// Either both were not specified (will be allocated) or both were
-		// specified (will be validated).
-		return
-	}
-
-	// This was an update operation
-
-	// ClusterIPs were cleared by an old client which was trying to patch
-	// some field and didn't provide ClusterIPs
-	if len(oldSvc.Spec.ClusterIPs) > 0 && len(newSvc.Spec.ClusterIPs) == 0 {
-		// if ClusterIP is the same, then it is an old client trying to
-		// patch service and didn't provide ClusterIPs
-		if oldSvc.Spec.ClusterIP == newSvc.Spec.ClusterIP {
-			newSvc.Spec.ClusterIPs = oldSvc.Spec.ClusterIPs
-		}
-	}
-
-	// clusterIP is not the same
-	if oldSvc.Spec.ClusterIP != newSvc.Spec.ClusterIP {
-		// this is a client trying to clear it
-		if len(oldSvc.Spec.ClusterIP) > 0 && len(newSvc.Spec.ClusterIP) == 0 {
-			// if clusterIPs are the same, then clear on their behalf
-			if sameStringSlice(oldSvc.Spec.ClusterIPs, newSvc.Spec.ClusterIPs) {
-				newSvc.Spec.ClusterIPs = nil
-			}
-
-			// if they provided nil, then we are fine (handled by patching case above)
-			// if they changed it then validation will catch it
-		} else {
-			// ClusterIP has changed but not cleared *and* ClusterIPs are the same
-			// then we set ClusterIPs based on ClusterIP
-			if sameStringSlice(oldSvc.Spec.ClusterIPs, newSvc.Spec.ClusterIPs) {
-				newSvc.Spec.ClusterIPs = []string{newSvc.Spec.ClusterIP}
-			}
-		}
-	}
-}
-
 func sameStringSlice(a []string, b []string) bool {
 	if len(a) != len(b) {
 		return false
diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go
index 0c9a0f31db2..6bb5d8d785d 100644
--- a/pkg/registry/core/service/strategy_test.go
+++ b/pkg/registry/core/service/strategy_test.go
@@ -115,15 +115,6 @@ func makeValidServiceCustom(tweaks ...func(svc *api.Service)) *api.Service {
 	return svc
 }
 
-func makeServiceWithClusterIp(clusterIP string, clusterIPs []string) *api.Service {
-	return &api.Service{
-		Spec: api.ServiceSpec{
-			ClusterIP:  clusterIP,
-			ClusterIPs: clusterIPs,
-		},
-	}
-}
-
 func TestServiceStatusStrategy(t *testing.T) {
 	_, testStatusStrategy := newStrategy("10.0.0.0/16", false)
 	ctx := genericapirequest.NewDefaultContext()
@@ -496,155 +487,6 @@ func TestDropDisabledField(t *testing.T) {
 
 }
 
-func TestNormalizeClusterIPs(t *testing.T) {
-	testCases := []struct {
-		name               string
-		oldService         *api.Service
-		newService         *api.Service
-		expectedClusterIP  string
-		expectedClusterIPs []string
-	}{
-		{
-			name:               "new - only clusterip used",
-			oldService:         nil,
-			newService:         makeServiceWithClusterIp("10.0.0.10", nil),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "new - only clusterips used",
-			oldService:         nil,
-			newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
-			expectedClusterIP:  "", // this is a validation issue, and validation will catch it
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "new - both used",
-			oldService:         nil,
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - no change",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - malformed change",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11"}),
-			expectedClusterIP:  "10.0.0.11",
-			expectedClusterIPs: []string{"10.0.0.11"},
-		},
-		{
-			name:               "update - malformed change on secondary ip",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11", "3000::1"}),
-			expectedClusterIP:  "10.0.0.11",
-			expectedClusterIPs: []string{"10.0.0.11", "3000::1"},
-		},
-		{
-			name:               "update - upgrade",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10", "2000::1"},
-		},
-		{
-			name:               "update - downgrade",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - user cleared cluster IP",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
-			expectedClusterIP:  "",
-			expectedClusterIPs: nil,
-		},
-		{
-			name:               "update - user cleared clusterIPs", // *MUST* REMAIN FOR OLD CLIENTS
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", nil),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - user cleared both",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("", nil),
-			expectedClusterIP:  "",
-			expectedClusterIPs: nil,
-		},
-		{
-			name:               "update - user cleared ClusterIP but changed clusterIPs",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("", []string{"10.0.0.11"}),
-			expectedClusterIP:  "", /* validation catches this */
-			expectedClusterIPs: []string{"10.0.0.11"},
-		},
-		{
-			name:               "update - user cleared ClusterIPs but changed ClusterIP",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("10.0.0.11", nil),
-			expectedClusterIP:  "10.0.0.11",
-			expectedClusterIPs: nil,
-		},
-		{
-			name:               "update - user changed from None to ClusterIP",
-			oldService:         makeServiceWithClusterIp("None", []string{"None"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"None"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - user changed from ClusterIP to None",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("None", []string{"10.0.0.10"}),
-			expectedClusterIP:  "None",
-			expectedClusterIPs: []string{"None"},
-		},
-		{
-			name:               "update - user changed from ClusterIP to None and changed ClusterIPs in a dual stack (new client making a mistake)",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("None", []string{"10.0.0.11", "2000::1"}),
-			expectedClusterIP:  "None",
-			expectedClusterIPs: []string{"10.0.0.11", "2000::1"},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			NormalizeClusterIPs(tc.oldService, tc.newService)
-
-			if tc.newService == nil {
-				t.Fatalf("unexpected new service to be nil")
-			}
-
-			if tc.newService.Spec.ClusterIP != tc.expectedClusterIP {
-				t.Fatalf("expected clusterIP [%v] got [%v]", tc.expectedClusterIP, tc.newService.Spec.ClusterIP)
-			}
-
-			if len(tc.newService.Spec.ClusterIPs) != len(tc.expectedClusterIPs) {
-				t.Fatalf("expected  clusterIPs %v got %v", tc.expectedClusterIPs, tc.newService.Spec.ClusterIPs)
-			}
-
-			for idx, clusterIP := range tc.newService.Spec.ClusterIPs {
-				if clusterIP != tc.expectedClusterIPs[idx] {
-					t.Fatalf("expected clusterIP [%v] at index[%v] got [%v]", tc.expectedClusterIPs[idx], idx, tc.newService.Spec.ClusterIPs[idx])
-
-				}
-			}
-		})
-	}
-
-}
-
 func TestDropTypeDependentFields(t *testing.T) {
 	// Tweaks used below.
 	setTypeExternalName := func(svc *api.Service) {

From 017a430dcd84bc88bd9340def7c394b09532dd68 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Mon, 23 Aug 2021 21:55:38 -0700
Subject: [PATCH 55/79] Svc REST: Move patchAllocatedValues to storage pkg

All the meaningful callers of it are in that pkg anyway.  Removes 1
FIXME.
---
 pkg/registry/core/service/storage/storage.go  |  85 +++++++++++-
 .../core/service/storage/storage_test.go      | 120 +++++++++++++++++
 pkg/registry/core/service/strategy.go         |  58 ---------
 pkg/registry/core/service/strategy_test.go    | 123 ------------------
 4 files changed, 204 insertions(+), 182 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 7ba08a78646..056005dac7d 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
@@ -354,7 +355,7 @@ func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Objec
 
 	// Fix up allocated values that the client may have not specified (for
 	// idempotence).
-	svcreg.PatchAllocatedValues(newSvc, oldSvc)
+	patchAllocatedValues(newSvc, oldSvc)
 
 	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
 	// early, before anyone looks at them.
@@ -514,3 +515,85 @@ func normalizeClusterIPs(oldSvc, newSvc *api.Service) {
 		}
 	}
 }
+
+// patchAllocatedValues allows clients to avoid a read-modify-write cycle while
+// preserving values that we allocated on their behalf.  For example, they
+// might create a Service without specifying the ClusterIP, in which case we
+// allocate one.  If they resubmit that same YAML, we want it to succeed.
+func patchAllocatedValues(newSvc, oldSvc *api.Service) {
+	if needsClusterIP(oldSvc) && needsClusterIP(newSvc) {
+		if newSvc.Spec.ClusterIP == "" {
+			newSvc.Spec.ClusterIP = oldSvc.Spec.ClusterIP
+		}
+		if len(newSvc.Spec.ClusterIPs) == 0 && len(oldSvc.Spec.ClusterIPs) > 0 {
+			newSvc.Spec.ClusterIPs = oldSvc.Spec.ClusterIPs
+		}
+	}
+
+	if needsNodePort(oldSvc) && needsNodePort(newSvc) {
+		nodePortsUsed := func(svc *api.Service) sets.Int32 {
+			used := sets.NewInt32()
+			for _, p := range svc.Spec.Ports {
+				if p.NodePort != 0 {
+					used.Insert(p.NodePort)
+				}
+			}
+			return used
+		}
+
+		// Build a set of all the ports in oldSvc that are also in newSvc.  We know
+		// we can't patch these values.
+		used := nodePortsUsed(oldSvc).Intersection(nodePortsUsed(newSvc))
+
+		// Map NodePorts by name.  The user may have changed other properties
+		// of the port, but we won't see that here.
+		np := map[string]int32{}
+		for i := range oldSvc.Spec.Ports {
+			p := &oldSvc.Spec.Ports[i]
+			np[p.Name] = p.NodePort
+		}
+
+		// If newSvc is missing values, try to patch them in when we know them and
+		// they haven't been used for another port.
+
+		for i := range newSvc.Spec.Ports {
+			p := &newSvc.Spec.Ports[i]
+			if p.NodePort == 0 {
+				oldVal := np[p.Name]
+				if !used.Has(oldVal) {
+					p.NodePort = oldVal
+				}
+			}
+		}
+	}
+
+	if needsHCNodePort(oldSvc) && needsHCNodePort(newSvc) {
+		if newSvc.Spec.HealthCheckNodePort == 0 {
+			newSvc.Spec.HealthCheckNodePort = oldSvc.Spec.HealthCheckNodePort
+		}
+	}
+}
+
+func needsClusterIP(svc *api.Service) bool {
+	if svc.Spec.Type == api.ServiceTypeExternalName {
+		return false
+	}
+	return true
+}
+
+func needsNodePort(svc *api.Service) bool {
+	if svc.Spec.Type == api.ServiceTypeNodePort || svc.Spec.Type == api.ServiceTypeLoadBalancer {
+		return true
+	}
+	return false
+}
+
+func needsHCNodePort(svc *api.Service) bool {
+	if svc.Spec.Type != api.ServiceTypeLoadBalancer {
+		return false
+	}
+	if svc.Spec.ExternalTrafficPolicy != api.ServiceExternalTrafficPolicyTypeLocal {
+		return false
+	}
+	return true
+}
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 652cb9db10f..2dadd00d68e 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -453,6 +453,126 @@ func TestNormalizeClusterIPs(t *testing.T) {
 	}
 }
 
+func TestPatchAllocatedValues(t *testing.T) {
+	testCases := []struct {
+		name                    string
+		before                  *api.Service
+		update                  *api.Service
+		expectSameClusterIPs    bool
+		expectReducedClusterIPs bool
+		expectSameNodePort      bool
+		expectSameHCNP          bool
+	}{{
+		name: "all_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+		expectSameClusterIPs: true,
+		expectSameNodePort:   true,
+		expectSameHCNP:       true,
+	}, {
+		name: "IPs_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP,
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			// these are not valid, but prove the test
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeClusterIP),
+		expectSameClusterIPs: true,
+	}, {
+		name: "NPs_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			// this is not valid, but proves the test
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeNodePort,
+			svctest.SetClusterIPs("10.0.0.93", "2000::76")),
+		expectSameClusterIPs: true,
+		expectSameNodePort:   true,
+	}, {
+		name: "HCNP_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts),
+		expectSameClusterIPs: true,
+		expectSameNodePort:   true,
+		expectSameHCNP:       true,
+	}, {
+		name: "nothing_patched",
+		before: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			// these are not valid, but prove the test
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
+			svctest.SetUniqueNodePorts,
+			svctest.SetHealthCheckNodePort(31234)),
+		update: svctest.MakeService("foo",
+			svctest.SetTypeExternalName,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			update := tc.update.DeepCopy()
+			patchAllocatedValues(update, tc.before)
+
+			beforeIP := tc.before.Spec.ClusterIP
+			updateIP := update.Spec.ClusterIP
+			if tc.expectSameClusterIPs || tc.expectReducedClusterIPs {
+				if beforeIP != updateIP {
+					t.Errorf("expected clusterIP to be patched: %q != %q", beforeIP, updateIP)
+				}
+			} else if beforeIP == updateIP {
+				t.Errorf("expected clusterIP to not be patched: %q == %q", beforeIP, updateIP)
+			}
+
+			beforeIPs := tc.before.Spec.ClusterIPs
+			updateIPs := update.Spec.ClusterIPs
+			if tc.expectSameClusterIPs {
+				if !cmp.Equal(beforeIPs, updateIPs) {
+					t.Errorf("expected clusterIPs to be patched: %q != %q", beforeIPs, updateIPs)
+				}
+			} else if tc.expectReducedClusterIPs {
+				if len(updateIPs) != 1 || beforeIPs[0] != updateIPs[0] {
+					t.Errorf("expected clusterIPs to be trim-patched: %q -> %q", beforeIPs, updateIPs)
+				}
+			} else if cmp.Equal(beforeIPs, updateIPs) {
+				t.Errorf("expected clusterIPs to not be patched: %q == %q", beforeIPs, updateIPs)
+			}
+			if b, u := tc.before.Spec.Ports[0].NodePort, update.Spec.Ports[0].NodePort; tc.expectSameNodePort && b != u {
+				t.Errorf("expected nodePort to be patched: %d != %d", b, u)
+			} else if !tc.expectSameNodePort && b == u {
+				t.Errorf("expected nodePort to not be patched: %d == %d", b, u)
+			}
+
+			if b, u := tc.before.Spec.HealthCheckNodePort, update.Spec.HealthCheckNodePort; tc.expectSameHCNP && b != u {
+				t.Errorf("expected healthCheckNodePort to be patched: %d != %d", b, u)
+			} else if !tc.expectSameHCNP && b == u {
+				t.Errorf("expected healthCheckNodePort to not be patched: %d == %d", b, u)
+			}
+		})
+	}
+}
+
 func TestServiceDefaultOnRead(t *testing.T) {
 	// Helper makes a mostly-valid ServiceList.  Test-cases can tweak it as needed.
 	makeServiceList := func(tweaks ...svctest.Tweak) *api.ServiceList {
diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go
index b4bd7fa45fd..473757a258d 100644
--- a/pkg/registry/core/service/strategy.go
+++ b/pkg/registry/core/service/strategy.go
@@ -299,64 +299,6 @@ func (serviceStatusStrategy) WarningsOnUpdate(ctx context.Context, obj, old runt
 	return nil
 }
 
-// PatchAllocatedValues allows clients to avoid a read-modify-write cycle while
-// preserving values that we allocated on their behalf.  For example, they
-// might create a Service without specifying the ClusterIP, in which case we
-// allocate one.  If they resubmit that same YAML, we want it to succeed.
-//FIXME: move this to pkg/registry/core/service/storage
-func PatchAllocatedValues(newSvc, oldSvc *api.Service) {
-	if needsClusterIP(oldSvc) && needsClusterIP(newSvc) {
-		if newSvc.Spec.ClusterIP == "" {
-			newSvc.Spec.ClusterIP = oldSvc.Spec.ClusterIP
-		}
-		if len(newSvc.Spec.ClusterIPs) == 0 && len(oldSvc.Spec.ClusterIPs) > 0 {
-			newSvc.Spec.ClusterIPs = oldSvc.Spec.ClusterIPs
-		}
-	}
-
-	if needsNodePort(oldSvc) && needsNodePort(newSvc) {
-		nodePortsUsed := func(svc *api.Service) sets.Int32 {
-			used := sets.NewInt32()
-			for _, p := range svc.Spec.Ports {
-				if p.NodePort != 0 {
-					used.Insert(p.NodePort)
-				}
-			}
-			return used
-		}
-
-		// Build a set of all the ports in oldSvc that are also in newSvc.  We know
-		// we can't patch these values.
-		used := nodePortsUsed(oldSvc).Intersection(nodePortsUsed(newSvc))
-
-		// Map NodePorts by name.  The user may have changed other properties
-		// of the port, but we won't see that here.
-		np := map[string]int32{}
-		for i := range oldSvc.Spec.Ports {
-			p := &oldSvc.Spec.Ports[i]
-			np[p.Name] = p.NodePort
-		}
-
-		// If newSvc is missing values, try to patch them in when we know them and
-		// they haven't been used for another port.
-		for i := range newSvc.Spec.Ports {
-			p := &newSvc.Spec.Ports[i]
-			if p.NodePort == 0 {
-				oldVal := np[p.Name]
-				if !used.Has(oldVal) {
-					p.NodePort = oldVal
-				}
-			}
-		}
-	}
-
-	if needsHCNodePort(oldSvc) && needsHCNodePort(newSvc) {
-		if newSvc.Spec.HealthCheckNodePort == 0 {
-			newSvc.Spec.HealthCheckNodePort = oldSvc.Spec.HealthCheckNodePort
-		}
-	}
-}
-
 func sameStringSlice(a []string, b []string) bool {
 	if len(a) != len(b) {
 		return false
diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go
index 6bb5d8d785d..da9e0d05461 100644
--- a/pkg/registry/core/service/strategy_test.go
+++ b/pkg/registry/core/service/strategy_test.go
@@ -20,7 +20,6 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
@@ -29,7 +28,6 @@ import (
 	"k8s.io/apiserver/pkg/registry/rest"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
-	svctest "k8s.io/kubernetes/pkg/api/service/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	_ "k8s.io/kubernetes/pkg/apis/core/install"
 	"k8s.io/kubernetes/pkg/features"
@@ -771,124 +769,3 @@ func TestDropTypeDependentFields(t *testing.T) {
 		})
 	}
 }
-
-func TestPatchAllocatedValues(t *testing.T) {
-	testCases := []struct {
-		name                    string
-		before                  *api.Service
-		update                  *api.Service
-		expectSameClusterIPs    bool
-		expectReducedClusterIPs bool
-		expectSameNodePort      bool
-		expectSameHCNP          bool
-	}{{
-		name: "all_patched",
-		before: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
-			svctest.SetUniqueNodePorts,
-			svctest.SetHealthCheckNodePort(31234)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-		expectSameClusterIPs: true,
-		expectSameNodePort:   true,
-		expectSameHCNP:       true,
-	}, {
-		name: "IPs_patched",
-		before: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP,
-			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
-			// these are not valid, but prove the test
-			svctest.SetUniqueNodePorts,
-			svctest.SetHealthCheckNodePort(31234)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeClusterIP),
-		expectSameClusterIPs: true,
-	}, {
-		name: "NPs_patched",
-		before: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
-			svctest.SetUniqueNodePorts,
-			// this is not valid, but proves the test
-			svctest.SetHealthCheckNodePort(31234)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeNodePort,
-			svctest.SetClusterIPs("10.0.0.93", "2000::76")),
-		expectSameClusterIPs: true,
-		expectSameNodePort:   true,
-	}, {
-		name: "HCNP_patched",
-		before: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
-			svctest.SetUniqueNodePorts,
-			svctest.SetHealthCheckNodePort(31234)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
-			svctest.SetUniqueNodePorts),
-		expectSameClusterIPs: true,
-		expectSameNodePort:   true,
-		expectSameHCNP:       true,
-	}, {
-		name: "nothing_patched",
-		before: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			// these are not valid, but prove the test
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetClusterIPs("10.0.0.93", "2000::76"),
-			svctest.SetUniqueNodePorts,
-			svctest.SetHealthCheckNodePort(31234)),
-		update: svctest.MakeService("foo",
-			svctest.SetTypeExternalName,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-	}}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			update := tc.update.DeepCopy()
-			PatchAllocatedValues(update, tc.before)
-
-			beforeIP := tc.before.Spec.ClusterIP
-			updateIP := update.Spec.ClusterIP
-			if tc.expectSameClusterIPs || tc.expectReducedClusterIPs {
-				if beforeIP != updateIP {
-					t.Errorf("expected clusterIP to be patched: %q != %q", beforeIP, updateIP)
-				}
-			} else if beforeIP == updateIP {
-				t.Errorf("expected clusterIP to not be patched: %q == %q", beforeIP, updateIP)
-			}
-
-			beforeIPs := tc.before.Spec.ClusterIPs
-			updateIPs := update.Spec.ClusterIPs
-			if tc.expectSameClusterIPs {
-				if !cmp.Equal(beforeIPs, updateIPs) {
-					t.Errorf("expected clusterIPs to be patched: %q != %q", beforeIPs, updateIPs)
-				}
-			} else if tc.expectReducedClusterIPs {
-				if len(updateIPs) != 1 || beforeIPs[0] != updateIPs[0] {
-					t.Errorf("expected clusterIPs to be trim-patched: %q -> %q", beforeIPs, updateIPs)
-				}
-			} else if cmp.Equal(beforeIPs, updateIPs) {
-				t.Errorf("expected clusterIPs to not be patched: %q == %q", beforeIPs, updateIPs)
-			}
-
-			if b, u := tc.before.Spec.Ports[0].NodePort, update.Spec.Ports[0].NodePort; tc.expectSameNodePort && b != u {
-				t.Errorf("expected nodePort to be patched: %d != %d", b, u)
-			} else if !tc.expectSameNodePort && b == u {
-				t.Errorf("expected nodePort to not be patched: %d == %d", b, u)
-			}
-
-			if b, u := tc.before.Spec.HealthCheckNodePort, update.Spec.HealthCheckNodePort; tc.expectSameHCNP && b != u {
-				t.Errorf("expected healthCheckNodePort to be patched: %d != %d", b, u)
-			} else if !tc.expectSameHCNP && b == u {
-				t.Errorf("expected healthCheckNodePort to not be patched: %d == %d", b, u)
-			}
-		})
-	}
-}

From 8f5189a49fd3d45dc0f4b3de0bdb3af531c3119e Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Mon, 23 Aug 2021 22:23:56 -0700
Subject: [PATCH 56/79] Svc REST: Move tests and scaffolding around

No code edits.  Just a little whitespace, adding comments, and
re-ordering functions.
---
 .../core/service/storage/storage_test.go      | 1399 +++++++++--------
 1 file changed, 701 insertions(+), 698 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 2dadd00d68e..a6000cc700f 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -53,41 +53,7 @@ import (
 	netutils "k8s.io/utils/net"
 )
 
-func makeIPAllocator(cidr *net.IPNet) ipallocator.Interface {
-	al, err := ipallocator.NewInMemory(cidr)
-	if err != nil {
-		panic(fmt.Sprintf("error creating IP allocator: %v", err))
-	}
-	return al
-}
-
-func makePortAllocator(ports machineryutilnet.PortRange) portallocator.Interface {
-	al, err := portallocator.NewInMemory(ports)
-	if err != nil {
-		panic(fmt.Sprintf("error creating port allocator: %v", err))
-	}
-	return al
-}
-
-func ipIsAllocated(t *testing.T, alloc ipallocator.Interface, ipstr string) bool {
-	t.Helper()
-	ip := netutils.ParseIPSloppy(ipstr)
-	if ip == nil {
-		t.Errorf("error parsing IP %q", ipstr)
-		return false
-	}
-	return alloc.Has(ip)
-}
-
-func portIsAllocated(t *testing.T, alloc portallocator.Interface, port int32) bool {
-	t.Helper()
-	if port == 0 {
-		t.Errorf("port is 0")
-		return false
-	}
-	return alloc.Has(int(port))
-}
-
+// Most tests will use this to create a registry to run tests against.
 func newStorage(t *testing.T, ipFamilies []api.IPFamily) (*wrapperRESTForTests, *StatusREST, *etcd3testing.EtcdTestServer) {
 	return newStorageWithPods(t, ipFamilies, nil, nil)
 }
@@ -162,6 +128,22 @@ func newStorageWithPods(t *testing.T, ipFamilies []api.IPFamily, pods []api.Pod,
 	return &wrapperRESTForTests{serviceStorage}, statusStorage, server
 }
 
+func makeIPAllocator(cidr *net.IPNet) ipallocator.Interface {
+	al, err := ipallocator.NewInMemory(cidr)
+	if err != nil {
+		panic(fmt.Sprintf("error creating IP allocator: %v", err))
+	}
+	return al
+}
+
+func makePortAllocator(ports machineryutilnet.PortRange) portallocator.Interface {
+	al, err := portallocator.NewInMemory(ports)
+	if err != nil {
+		panic(fmt.Sprintf("error creating port allocator: %v", err))
+	}
+	return al
+}
+
 // wrapperRESTForTests is a *trivial* wrapper for the real REST, which allows us to do
 // things that are specifically to enhance test safety.
 type wrapperRESTForTests struct {
@@ -176,6 +158,10 @@ func (f *wrapperRESTForTests) Create(ctx context.Context, obj runtime.Object, cr
 	return f.GenericREST.Create(ctx, obj, createValidation, options)
 }
 
+//
+// Generic registry tests
+//
+
 // This is used in generic registry tests.
 func validService() *api.Service {
 	return svctest.MakeService("foo",
@@ -296,6 +282,10 @@ func TestGenericCategories(t *testing.T) {
 	registrytest.AssertCategories(t, storage, expected)
 }
 
+//
+// Tests of internal functions
+//
+
 func TestNormalizeClusterIPs(t *testing.T) {
 	makeServiceWithClusterIp := func(clusterIP string, clusterIPs []string) *api.Service {
 		return &api.Service{
@@ -312,120 +302,103 @@ func TestNormalizeClusterIPs(t *testing.T) {
 		newService         *api.Service
 		expectedClusterIP  string
 		expectedClusterIPs []string
-	}{
-		{
-			name:               "new - only clusterip used",
-			oldService:         nil,
-			newService:         makeServiceWithClusterIp("10.0.0.10", nil),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "new - only clusterips used",
-			oldService:         nil,
-			newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
-			expectedClusterIP:  "", // this is a validation issue, and validation will catch it
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "new - both used",
-			oldService:         nil,
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - no change",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - malformed change",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11"}),
-			expectedClusterIP:  "10.0.0.11",
-			expectedClusterIPs: []string{"10.0.0.11"},
-		},
-		{
-			name:               "update - malformed change on secondary ip",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11", "3000::1"}),
-			expectedClusterIP:  "10.0.0.11",
-			expectedClusterIPs: []string{"10.0.0.11", "3000::1"},
-		},
-		{
-			name:               "update - upgrade",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10", "2000::1"},
-		},
-		{
-			name:               "update - downgrade",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - user cleared cluster IP",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
-			expectedClusterIP:  "",
-			expectedClusterIPs: nil,
-		},
-		{
-			name:               "update - user cleared clusterIPs", // *MUST* REMAIN FOR OLD CLIENTS
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", nil),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - user cleared both",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("", nil),
-			expectedClusterIP:  "",
-			expectedClusterIPs: nil,
-		},
-		{
-			name:               "update - user cleared ClusterIP but changed clusterIPs",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("", []string{"10.0.0.11"}),
-			expectedClusterIP:  "", /* validation catches this */
-			expectedClusterIPs: []string{"10.0.0.11"},
-		},
-		{
-			name:               "update - user cleared ClusterIPs but changed ClusterIP",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("10.0.0.11", nil),
-			expectedClusterIP:  "10.0.0.11",
-			expectedClusterIPs: nil,
-		},
-		{
-			name:               "update - user changed from None to ClusterIP",
-			oldService:         makeServiceWithClusterIp("None", []string{"None"}),
-			newService:         makeServiceWithClusterIp("10.0.0.10", []string{"None"}),
-			expectedClusterIP:  "10.0.0.10",
-			expectedClusterIPs: []string{"10.0.0.10"},
-		},
-		{
-			name:               "update - user changed from ClusterIP to None",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
-			newService:         makeServiceWithClusterIp("None", []string{"10.0.0.10"}),
-			expectedClusterIP:  "None",
-			expectedClusterIPs: []string{"None"},
-		},
-		{
-			name:               "update - user changed from ClusterIP to None and changed ClusterIPs in a dual stack (new client making a mistake)",
-			oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
-			newService:         makeServiceWithClusterIp("None", []string{"10.0.0.11", "2000::1"}),
-			expectedClusterIP:  "None",
-			expectedClusterIPs: []string{"10.0.0.11", "2000::1"},
-		},
-	}
+	}{{
+		name:               "new - only clusterip used",
+		oldService:         nil,
+		newService:         makeServiceWithClusterIp("10.0.0.10", nil),
+		expectedClusterIP:  "10.0.0.10",
+		expectedClusterIPs: []string{"10.0.0.10"},
+	}, {
+		name:               "new - only clusterips used",
+		oldService:         nil,
+		newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
+		expectedClusterIP:  "", // this is a validation issue, and validation will catch it
+		expectedClusterIPs: []string{"10.0.0.10"},
+	}, {
+		name:               "new - both used",
+		oldService:         nil,
+		newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		expectedClusterIP:  "10.0.0.10",
+		expectedClusterIPs: []string{"10.0.0.10"},
+	}, {
+		name:               "update - no change",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		expectedClusterIP:  "10.0.0.10",
+		expectedClusterIPs: []string{"10.0.0.10"},
+	}, {
+		name:               "update - malformed change",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11"}),
+		expectedClusterIP:  "10.0.0.11",
+		expectedClusterIPs: []string{"10.0.0.11"},
+	}, {
+		name:               "update - malformed change on secondary ip",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+		newService:         makeServiceWithClusterIp("10.0.0.11", []string{"10.0.0.11", "3000::1"}),
+		expectedClusterIP:  "10.0.0.11",
+		expectedClusterIPs: []string{"10.0.0.11", "3000::1"},
+	}, {
+		name:               "update - upgrade",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+		expectedClusterIP:  "10.0.0.10",
+		expectedClusterIPs: []string{"10.0.0.10", "2000::1"},
+	}, {
+		name:               "update - downgrade",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+		newService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		expectedClusterIP:  "10.0.0.10",
+		expectedClusterIPs: []string{"10.0.0.10"},
+	}, {
+		name:               "update - user cleared cluster IP",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("", []string{"10.0.0.10"}),
+		expectedClusterIP:  "",
+		expectedClusterIPs: nil,
+	}, {
+		name:               "update - user cleared clusterIPs", // *MUST* REMAIN FOR OLD CLIENTS
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("10.0.0.10", nil),
+		expectedClusterIP:  "10.0.0.10",
+		expectedClusterIPs: []string{"10.0.0.10"},
+	}, {
+		name:               "update - user cleared both",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("", nil),
+		expectedClusterIP:  "",
+		expectedClusterIPs: nil,
+	}, {
+		name:               "update - user cleared ClusterIP but changed clusterIPs",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("", []string{"10.0.0.11"}),
+		expectedClusterIP:  "", /* validation catches this */
+		expectedClusterIPs: []string{"10.0.0.11"},
+	}, {
+		name:               "update - user cleared ClusterIPs but changed ClusterIP",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+		newService:         makeServiceWithClusterIp("10.0.0.11", nil),
+		expectedClusterIP:  "10.0.0.11",
+		expectedClusterIPs: nil,
+	}, {
+		name:               "update - user changed from None to ClusterIP",
+		oldService:         makeServiceWithClusterIp("None", []string{"None"}),
+		newService:         makeServiceWithClusterIp("10.0.0.10", []string{"None"}),
+		expectedClusterIP:  "10.0.0.10",
+		expectedClusterIPs: []string{"10.0.0.10"},
+	}, {
+		name:               "update - user changed from ClusterIP to None",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10"}),
+		newService:         makeServiceWithClusterIp("None", []string{"10.0.0.10"}),
+		expectedClusterIP:  "None",
+		expectedClusterIPs: []string{"None"},
+	}, {
+		name:               "update - user changed from ClusterIP to None and changed ClusterIPs in a dual stack (new client making a mistake)",
+		oldService:         makeServiceWithClusterIp("10.0.0.10", []string{"10.0.0.10", "2000::1"}),
+		newService:         makeServiceWithClusterIp("None", []string{"10.0.0.11", "2000::1"}),
+		expectedClusterIP:  "None",
+		expectedClusterIPs: []string{"10.0.0.11", "2000::1"},
+	}}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -656,6 +629,584 @@ func TestServiceDefaultOnRead(t *testing.T) {
 	}
 }
 
+//
+// Scaffolding for create-update-delete tests.  Many tests can and should be
+// written in terms of this.
+//
+
+type cudTestCase struct {
+	name         string
+	line         string // if not empty, will be logged with errors, use line() to set
+	create       svcTestCase
+	beforeUpdate func(t *testing.T, storage *wrapperRESTForTests)
+	update       svcTestCase
+}
+
+type svcTestCase struct {
+	svc         *api.Service
+	expectError bool
+
+	// We could calculate these by looking at the Service, but that's a
+	// vector for test bugs and more importantly it makes the test cases less
+	// self-documenting.
+	expectClusterIPs          bool
+	expectStackDowngrade      bool
+	expectHeadless            bool
+	expectNodePorts           bool
+	expectHealthCheckNodePort bool
+
+	// Additional proofs, provided by the tests which use this.
+	prove []svcTestProof
+}
+
+type svcTestProof func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service)
+
+// Most tests will call this.
+func helpTestCreateUpdateDelete(t *testing.T, testCases []cudTestCase) {
+	t.Helper()
+	helpTestCreateUpdateDeleteWithFamilies(t, testCases, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
+}
+
+func helpTestCreateUpdateDeleteWithFamilies(t *testing.T, testCases []cudTestCase, ipFamilies []api.IPFamily) {
+	// NOTE: do not call t.Helper() here.  It's more useful for errors to be
+	// attributed to lines in this function than the caller of it.
+
+	// This test is ONLY with the gate enabled.
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	storage, _, server := newStorage(t, ipFamilies)
+	defer server.Terminate(t)
+	defer storage.Store.DestroyFunc()
+
+	for _, tc := range testCases {
+		name := tc.name
+		if tc.line != "" {
+			name += "__@L" + tc.line
+		}
+		t.Run(name, func(t *testing.T) {
+			ctx := genericapirequest.NewDefaultContext()
+
+			// Create the object as specified and check the results.
+			obj, err := storage.Create(ctx, tc.create.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
+			if tc.create.expectError && err != nil {
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error creating service: %v", err)
+			}
+			defer storage.Delete(ctx, tc.create.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) // in case
+			if tc.create.expectError && err == nil {
+				t.Fatalf("unexpected success creating service")
+			}
+			createdSvc := obj.(*api.Service)
+			if !verifyEquiv(t, "create", &tc.create, createdSvc) {
+				return
+			}
+			verifyExpectations(t, storage, tc.create, tc.create.svc, createdSvc)
+			lastSvc := createdSvc
+
+			// The update phase is optional.
+			if tc.update.svc != nil {
+				// Allow callers to do something between create and update.
+				if tc.beforeUpdate != nil {
+					tc.beforeUpdate(t, storage)
+				}
+
+				// Update the object to the new state and check the results.
+				obj, created, err := storage.Update(ctx, tc.update.svc.Name,
+					rest.DefaultUpdatedObjectInfo(tc.update.svc), rest.ValidateAllObjectFunc,
+					rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
+				if tc.update.expectError && err != nil {
+					return
+				}
+				if err != nil {
+					t.Fatalf("unexpected error updating service: %v", err)
+				}
+				if tc.update.expectError && err == nil {
+					t.Fatalf("unexpected success updating service")
+				}
+				if created {
+					t.Fatalf("unexpected create-on-update")
+				}
+				updatedSvc := obj.(*api.Service)
+				if !verifyEquiv(t, "update", &tc.update, updatedSvc) {
+					return
+				}
+				verifyExpectations(t, storage, tc.update, createdSvc, updatedSvc)
+				lastSvc = updatedSvc
+			}
+
+			// Delete the object and check the results.
+			_, _, err = storage.Delete(ctx, tc.create.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
+			if err != nil {
+				t.Fatalf("unexpected error deleting service: %v", err)
+			}
+			verifyExpectations(t, storage, svcTestCase{ /* all false */ }, lastSvc, nil)
+		})
+	}
+}
+
+// line returns the line number of the caller, if possible.  This is useful in
+// tests with a large number of cases - when something goes wrong you can find
+// which case more easily.
+func line() string {
+	_, _, line, ok := stdruntime.Caller(1)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%d", line)
+	} else {
+		s = "<??>"
+	}
+	return s
+}
+
+// This makes the test-helpers testable.
+type testingTInterface interface {
+	Helper()
+	Errorf(format string, args ...interface{})
+}
+
+type fakeTestingT struct {
+	t *testing.T
+}
+
+func (f fakeTestingT) Helper() {}
+
+func (f fakeTestingT) Errorf(format string, args ...interface{}) {
+	f.t.Logf(format, args...)
+}
+
+func verifyEquiv(t testingTInterface, call string, tc *svcTestCase, got *api.Service) bool {
+	t.Helper()
+
+	// For when we compare objects.
+	options := []cmp.Option{
+		// These are system-assigned values, we don't need to compare them.
+		cmpopts.IgnoreFields(api.Service{}, "UID", "ResourceVersion", "CreationTimestamp"),
+		// Treat nil slices and empty slices as the same (e.g. clusterIPs).
+		cmpopts.EquateEmpty(),
+	}
+
+	// For allocated fields, we want to be able to compare cleanly whether the
+	// input specified values or not.
+	want := tc.svc.DeepCopy()
+	if tc.expectClusterIPs || tc.expectHeadless {
+		if want.Spec.ClusterIP == "" {
+			want.Spec.ClusterIP = got.Spec.ClusterIP
+		}
+		if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+			if want.Spec.IPFamilyPolicy == nil {
+				want.Spec.IPFamilyPolicy = got.Spec.IPFamilyPolicy
+			}
+			if tc.expectStackDowngrade && len(want.Spec.ClusterIPs) > len(got.Spec.ClusterIPs) {
+				want.Spec.ClusterIPs = want.Spec.ClusterIPs[0:1]
+			} else if len(got.Spec.ClusterIPs) > len(want.Spec.ClusterIPs) {
+				want.Spec.ClusterIPs = append(want.Spec.ClusterIPs, got.Spec.ClusterIPs[len(want.Spec.ClusterIPs):]...)
+			}
+			if tc.expectStackDowngrade && len(want.Spec.IPFamilies) > len(got.Spec.ClusterIPs) {
+				want.Spec.IPFamilies = want.Spec.IPFamilies[0:1]
+			} else if len(got.Spec.IPFamilies) > len(want.Spec.IPFamilies) {
+				want.Spec.IPFamilies = append(want.Spec.IPFamilies, got.Spec.IPFamilies[len(want.Spec.IPFamilies):]...)
+			}
+		}
+	}
+
+	if tc.expectNodePorts {
+		for i := range want.Spec.Ports {
+			p := &want.Spec.Ports[i]
+			if p.NodePort == 0 {
+				p.NodePort = got.Spec.Ports[i].NodePort
+			}
+		}
+	}
+	if tc.expectHealthCheckNodePort {
+		if want.Spec.HealthCheckNodePort == 0 {
+			want.Spec.HealthCheckNodePort = got.Spec.HealthCheckNodePort
+		}
+	}
+
+	if !cmp.Equal(want, got, options...) {
+		t.Errorf("unexpected result from %s:\n%s", call, cmp.Diff(want, got, options...))
+		return false
+	}
+	return true
+}
+
+// Quis custodiet ipsos custodes?
+func TestVerifyEquiv(t *testing.T) {
+	testCases := []struct {
+		name   string
+		input  svcTestCase
+		output *api.Service
+		expect bool
+	}{{
+		name: "ExternalName",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeExternalName),
+		expect: true,
+	}, {
+		name: "ClusterIPs_unspecified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: true,
+	}, {
+		name: "ClusterIPs_specified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: true,
+	}, {
+		name: "ClusterIPs_wrong",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.0", "2000:0")),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: false,
+	}, {
+		name: "ClusterIPs_partial",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1")),
+			expectClusterIPs: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
+		expect: true,
+	}, {
+		name: "NodePort_unspecified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetUniqueNodePorts),
+		expect: true,
+	}, {
+		name: "NodePort_specified",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
+		expect: true,
+	}, {
+		name: "NodePort_wrong",
+		input: svcTestCase{
+			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(76)),
+		expect: false,
+	}, {
+		name: "NodePort_partial",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
+				svctest.SetPorts(
+					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+				svctest.SetNodePorts(93)),
+			expectClusterIPs: true,
+			expectNodePorts:  true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeNodePort,
+			svctest.SetPorts(
+				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
+				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
+			svctest.SetNodePorts(93, 76)),
+		expect: true,
+	}, {
+		name: "HealthCheckNodePort_unspecified",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(93)),
+		expect: true,
+	}, {
+		name: "HealthCheckNodePort_specified",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(93)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(93)),
+		expect: true,
+	}, {
+		name: "HealthCheckNodePort_wrong",
+		input: svcTestCase{
+			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+				svctest.SetHealthCheckNodePort(93)),
+			expectClusterIPs:          true,
+			expectNodePorts:           true,
+			expectHealthCheckNodePort: true,
+		},
+		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
+			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
+			svctest.SetHealthCheckNodePort(76)),
+		expect: false,
+	}}
+
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := verifyEquiv(fakeTestingT{t}, "test", &tc.input, tc.output)
+			if result != tc.expect {
+				t.Errorf("expected %v, got %v", tc.expect, result)
+			}
+		})
+	}
+}
+
+func verifyExpectations(t *testing.T, storage *wrapperRESTForTests, tc svcTestCase, before, after *api.Service) {
+	t.Helper()
+
+	if tc.expectClusterIPs {
+		proveClusterIPsAllocated(t, storage, before, after)
+	} else if tc.expectHeadless {
+		proveHeadless(t, storage, before, after)
+	} else {
+		proveClusterIPsDeallocated(t, storage, before, after)
+	}
+	if tc.expectNodePorts {
+		proveNodePortsAllocated(t, storage, before, after)
+	} else {
+		proveNodePortsDeallocated(t, storage, before, after)
+	}
+	if tc.expectHealthCheckNodePort {
+		proveHealthCheckNodePortAllocated(t, storage, before, after)
+	} else {
+		proveHealthCheckNodePortDeallocated(t, storage, before, after)
+	}
+
+	for _, p := range tc.prove {
+		p(t, storage, before, after)
+	}
+}
+
+func callName(before, after *api.Service) string {
+	if before == nil && after != nil {
+		return "create"
+	}
+	if before != nil && after != nil {
+		return "update"
+	}
+	if before != nil && after == nil {
+		return "delete"
+	}
+	panic("this test is broken: before and after are both nil")
+}
+
+func ipIsAllocated(t *testing.T, alloc ipallocator.Interface, ipstr string) bool {
+	t.Helper()
+	ip := netutils.ParseIPSloppy(ipstr)
+	if ip == nil {
+		t.Errorf("error parsing IP %q", ipstr)
+		return false
+	}
+	return alloc.Has(ip)
+}
+
+func portIsAllocated(t *testing.T, alloc portallocator.Interface, port int32) bool {
+	t.Helper()
+	if port == 0 {
+		t.Errorf("port is 0")
+		return false
+	}
+	return alloc.Has(int(port))
+}
+
+func proveClusterIPsAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
+	t.Helper()
+
+	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
+		t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
+	}
+
+	clips := []string{}
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		clips = after.Spec.ClusterIPs
+	} else {
+		clips = append(clips, after.Spec.ClusterIP)
+	}
+	for _, clip := range clips {
+		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
+			t.Errorf("%s: expected clusterIP to be allocated: %q", callName(before, after), clip)
+		}
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		if lc, lf := len(after.Spec.ClusterIPs), len(after.Spec.IPFamilies); lc != lf {
+			t.Errorf("%s: expected same number of clusterIPs and ipFamilies: %d != %d", callName(before, after), lc, lf)
+		}
+	}
+
+	for i, fam := range after.Spec.IPFamilies {
+		if want, got := fam, familyOf(after.Spec.ClusterIPs[i]); want != got {
+			t.Errorf("%s: clusterIP is the wrong IP family: want %s, got %s", callName(before, after), want, got)
+		}
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		if after.Spec.IPFamilyPolicy == nil {
+			t.Errorf("%s: expected ipFamilyPolicy to be set", callName(before, after))
+		} else {
+			pol := *after.Spec.IPFamilyPolicy
+			fams := len(after.Spec.IPFamilies)
+			clus := 1
+			if storage.secondaryIPFamily != "" {
+				clus = 2
+			}
+			if pol == api.IPFamilyPolicySingleStack && fams != 1 {
+				t.Errorf("%s: expected 1 ipFamily, got %d", callName(before, after), fams)
+			} else if pol == api.IPFamilyPolicyRequireDualStack && fams != 2 {
+				t.Errorf("%s: expected 2 ipFamilies, got %d", callName(before, after), fams)
+			} else if pol == api.IPFamilyPolicyPreferDualStack && fams != clus {
+				t.Errorf("%s: expected %d ipFamilies, got %d", callName(before, after), clus, fams)
+			}
+		}
+	}
+
+	if before != nil {
+		if before.Spec.ClusterIP != "" {
+			if want, got := before.Spec.ClusterIP, after.Spec.ClusterIP; want != got {
+				t.Errorf("%s: wrong clusterIP: wanted %q, got %q", callName(before, after), want, got)
+			}
+		}
+		min := func(x, y int) int {
+			if x < y {
+				return x
+			}
+			return y
+		}
+		for i := 0; i < min(len(before.Spec.ClusterIPs), len(after.Spec.ClusterIPs)); i++ {
+			if want, got := before.Spec.ClusterIPs[i], after.Spec.ClusterIPs[i]; want != got {
+				t.Errorf("%s: wrong clusterIPs[%d]: wanted %q, got %q", callName(before, after), i, want, got)
+			}
+		}
+		for i := 0; i < min(len(before.Spec.IPFamilies), len(after.Spec.IPFamilies)); i++ {
+			if want, got := before.Spec.IPFamilies[i], after.Spec.IPFamilies[i]; want != got {
+				t.Errorf("%s: wrong ipFamilies[%d]: wanted %q, got %q", callName(before, after), i, want, got)
+			}
+		}
+	}
+}
+
+func proveClusterIPsDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
+	t.Helper()
+
+	if after != nil && after.Spec.ClusterIP != api.ClusterIPNone {
+		if after.Spec.ClusterIP != "" {
+			t.Errorf("%s: expected clusterIP to be unset: %q", callName(before, after), after.Spec.ClusterIP)
+		}
+		if len(after.Spec.ClusterIPs) != 0 {
+			t.Errorf("%s: expected clusterIPs to be unset: %q", callName(before, after), after.Spec.ClusterIPs)
+		}
+	}
+
+	if before != nil && before.Spec.ClusterIP != api.ClusterIPNone {
+		clips := []string{}
+		if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+			clips = before.Spec.ClusterIPs
+		} else {
+			clips = append(clips, before.Spec.ClusterIP)
+		}
+		for _, clip := range clips {
+			if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
+				t.Errorf("%s: expected clusterIP to be deallocated: %q", callName(before, after), clip)
+			}
+		}
+	}
+}
+
+func proveHeadless(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
+	t.Helper()
+
+	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
+		t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
+	}
+	if len(after.Spec.ClusterIPs) != 1 || after.Spec.ClusterIPs[0] != api.ClusterIPNone {
+		t.Errorf("%s: expected clusterIPs to be [%q]: %q", callName(before, after), api.ClusterIPNone, after.Spec.ClusterIPs)
+	}
+}
+
+func proveNodePortsAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
+	t.Helper()
+
+	for _, p := range after.Spec.Ports {
+		if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+			t.Errorf("%s: expected nodePort to be allocated: %d", callName(before, after), p.NodePort)
+		}
+	}
+}
+
+func proveNodePortsDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
+	t.Helper()
+
+	if after != nil {
+		for _, p := range after.Spec.Ports {
+			if p.NodePort != 0 {
+				t.Errorf("%s: expected nodePort to be unset: %d", callName(before, after), p.NodePort)
+			}
+		}
+	}
+
+	if before != nil {
+		for _, p := range before.Spec.Ports {
+			if p.NodePort != 0 && portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
+				t.Errorf("%s: expected nodePort to be deallocated: %d", callName(before, after), p.NodePort)
+			}
+		}
+	}
+}
+
+func proveHealthCheckNodePortAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
+	t.Helper()
+
+	if !portIsAllocated(t, storage.alloc.serviceNodePorts, after.Spec.HealthCheckNodePort) {
+		t.Errorf("%s: expected healthCheckNodePort to be allocated: %d", callName(before, after), after.Spec.HealthCheckNodePort)
+	}
+}
+
+func proveHealthCheckNodePortDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
+	t.Helper()
+
+	if after != nil {
+		if after.Spec.HealthCheckNodePort != 0 {
+			t.Errorf("%s: expected healthCheckNodePort to be unset: %d", callName(before, after), after.Spec.HealthCheckNodePort)
+		}
+	}
+
+	if before != nil {
+		if before.Spec.HealthCheckNodePort != 0 && portIsAllocated(t, storage.alloc.serviceNodePorts, before.Spec.HealthCheckNodePort) {
+			t.Errorf("%s: expected healthCheckNodePort to be deallocated: %d", callName(before, after), before.Spec.HealthCheckNodePort)
+		}
+	}
+}
+
+//
+// functional tests of the registry
+//
+
 func fmtIPFamilyPolicy(pol *api.IPFamilyPolicyType) string {
 	if pol == nil {
 		return "<nil>"
@@ -1063,20 +1614,6 @@ func TestCreateInitClusterIPsFromClusterIP(t *testing.T) {
 	}
 }
 
-// line returns the line number of the caller, if possible.  This is useful in
-// tests with a large number of cases - when something goes wrong you can find
-// which case more easily.
-func line() string {
-	_, _, line, ok := stdruntime.Caller(1)
-	var s string
-	if ok {
-		s = fmt.Sprintf("%d", line)
-	} else {
-		s = "<??>"
-	}
-	return s
-}
-
 // Prove that create initializes IPFamily fields correctly.
 func TestCreateInitIPFields(t *testing.T) {
 	type testCase struct {
@@ -5678,6 +6215,7 @@ func TestCreateInvalidClusterIPInputs(t *testing.T) {
 		})
 	}
 }
+
 func TestCreateDeleteReuse(t *testing.T) {
 	testCases := []struct {
 		name string
@@ -10163,541 +10701,6 @@ func TestUpdateIPsFromDualStack(t *testing.T) {
 	})
 }
 
-//FIXME: after all the tests are ported, move this all up near the top?
-type svcTestCase struct {
-	svc         *api.Service
-	expectError bool
-
-	// We could calculate these by looking at the Service, but that's a
-	// vector for test bugs and more importantly it makes the test cases less
-	// self-documenting.
-	expectClusterIPs          bool
-	expectStackDowngrade      bool
-	expectHeadless            bool
-	expectNodePorts           bool
-	expectHealthCheckNodePort bool
-
-	// Additional proofs, provided by the tests which use this.
-	prove []svcTestProof
-}
-
-type svcTestProof func(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service)
-
-func callName(before, after *api.Service) string {
-	if before == nil && after != nil {
-		return "create"
-	}
-	if before != nil && after != nil {
-		return "update"
-	}
-	if before != nil && after == nil {
-		return "delete"
-	}
-	panic("this test is broken: before and after are both nil")
-}
-
-func proveClusterIPsAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
-	t.Helper()
-
-	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
-		t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
-	}
-
-	clips := []string{}
-	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		clips = after.Spec.ClusterIPs
-	} else {
-		clips = append(clips, after.Spec.ClusterIP)
-	}
-	for _, clip := range clips {
-		if !ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
-			t.Errorf("%s: expected clusterIP to be allocated: %q", callName(before, after), clip)
-		}
-	}
-
-	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		if lc, lf := len(after.Spec.ClusterIPs), len(after.Spec.IPFamilies); lc != lf {
-			t.Errorf("%s: expected same number of clusterIPs and ipFamilies: %d != %d", callName(before, after), lc, lf)
-		}
-	}
-
-	for i, fam := range after.Spec.IPFamilies {
-		if want, got := fam, familyOf(after.Spec.ClusterIPs[i]); want != got {
-			t.Errorf("%s: clusterIP is the wrong IP family: want %s, got %s", callName(before, after), want, got)
-		}
-	}
-
-	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		if after.Spec.IPFamilyPolicy == nil {
-			t.Errorf("%s: expected ipFamilyPolicy to be set", callName(before, after))
-		} else {
-			pol := *after.Spec.IPFamilyPolicy
-			fams := len(after.Spec.IPFamilies)
-			clus := 1
-			if storage.secondaryIPFamily != "" {
-				clus = 2
-			}
-			if pol == api.IPFamilyPolicySingleStack && fams != 1 {
-				t.Errorf("%s: expected 1 ipFamily, got %d", callName(before, after), fams)
-			} else if pol == api.IPFamilyPolicyRequireDualStack && fams != 2 {
-				t.Errorf("%s: expected 2 ipFamilies, got %d", callName(before, after), fams)
-			} else if pol == api.IPFamilyPolicyPreferDualStack && fams != clus {
-				t.Errorf("%s: expected %d ipFamilies, got %d", callName(before, after), clus, fams)
-			}
-		}
-	}
-
-	if before != nil {
-		if before.Spec.ClusterIP != "" {
-			if want, got := before.Spec.ClusterIP, after.Spec.ClusterIP; want != got {
-				t.Errorf("%s: wrong clusterIP: wanted %q, got %q", callName(before, after), want, got)
-			}
-		}
-		min := func(x, y int) int {
-			if x < y {
-				return x
-			}
-			return y
-		}
-		for i := 0; i < min(len(before.Spec.ClusterIPs), len(after.Spec.ClusterIPs)); i++ {
-			if want, got := before.Spec.ClusterIPs[i], after.Spec.ClusterIPs[i]; want != got {
-				t.Errorf("%s: wrong clusterIPs[%d]: wanted %q, got %q", callName(before, after), i, want, got)
-			}
-		}
-		for i := 0; i < min(len(before.Spec.IPFamilies), len(after.Spec.IPFamilies)); i++ {
-			if want, got := before.Spec.IPFamilies[i], after.Spec.IPFamilies[i]; want != got {
-				t.Errorf("%s: wrong ipFamilies[%d]: wanted %q, got %q", callName(before, after), i, want, got)
-			}
-		}
-	}
-}
-
-func proveClusterIPsDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
-	t.Helper()
-
-	if after != nil && after.Spec.ClusterIP != api.ClusterIPNone {
-		if after.Spec.ClusterIP != "" {
-			t.Errorf("%s: expected clusterIP to be unset: %q", callName(before, after), after.Spec.ClusterIP)
-		}
-		if len(after.Spec.ClusterIPs) != 0 {
-			t.Errorf("%s: expected clusterIPs to be unset: %q", callName(before, after), after.Spec.ClusterIPs)
-		}
-	}
-
-	if before != nil && before.Spec.ClusterIP != api.ClusterIPNone {
-		clips := []string{}
-		if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-			clips = before.Spec.ClusterIPs
-		} else {
-			clips = append(clips, before.Spec.ClusterIP)
-		}
-		for _, clip := range clips {
-			if ipIsAllocated(t, storage.alloc.serviceIPAllocatorsByFamily[familyOf(clip)], clip) {
-				t.Errorf("%s: expected clusterIP to be deallocated: %q", callName(before, after), clip)
-			}
-		}
-	}
-}
-
-func proveHeadless(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
-	t.Helper()
-
-	if sing, plur := after.Spec.ClusterIP, after.Spec.ClusterIPs[0]; sing != plur {
-		t.Errorf("%s: expected clusterIP == clusterIPs[0]: %q != %q", callName(before, after), sing, plur)
-	}
-	if len(after.Spec.ClusterIPs) != 1 || after.Spec.ClusterIPs[0] != api.ClusterIPNone {
-		t.Errorf("%s: expected clusterIPs to be [%q]: %q", callName(before, after), api.ClusterIPNone, after.Spec.ClusterIPs)
-	}
-}
-
-func proveNodePortsAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
-	t.Helper()
-
-	for _, p := range after.Spec.Ports {
-		if !portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-			t.Errorf("%s: expected nodePort to be allocated: %d", callName(before, after), p.NodePort)
-		}
-	}
-}
-
-func proveNodePortsDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
-	t.Helper()
-
-	if after != nil {
-		for _, p := range after.Spec.Ports {
-			if p.NodePort != 0 {
-				t.Errorf("%s: expected nodePort to be unset: %d", callName(before, after), p.NodePort)
-			}
-		}
-	}
-
-	if before != nil {
-		for _, p := range before.Spec.Ports {
-			if p.NodePort != 0 && portIsAllocated(t, storage.alloc.serviceNodePorts, p.NodePort) {
-				t.Errorf("%s: expected nodePort to be deallocated: %d", callName(before, after), p.NodePort)
-			}
-		}
-	}
-}
-
-func proveHealthCheckNodePortAllocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
-	t.Helper()
-
-	if !portIsAllocated(t, storage.alloc.serviceNodePorts, after.Spec.HealthCheckNodePort) {
-		t.Errorf("%s: expected healthCheckNodePort to be allocated: %d", callName(before, after), after.Spec.HealthCheckNodePort)
-	}
-}
-
-func proveHealthCheckNodePortDeallocated(t *testing.T, storage *wrapperRESTForTests, before, after *api.Service) {
-	t.Helper()
-
-	if after != nil {
-		if after.Spec.HealthCheckNodePort != 0 {
-			t.Errorf("%s: expected healthCheckNodePort to be unset: %d", callName(before, after), after.Spec.HealthCheckNodePort)
-		}
-	}
-
-	if before != nil {
-		if before.Spec.HealthCheckNodePort != 0 && portIsAllocated(t, storage.alloc.serviceNodePorts, before.Spec.HealthCheckNodePort) {
-			t.Errorf("%s: expected healthCheckNodePort to be deallocated: %d", callName(before, after), before.Spec.HealthCheckNodePort)
-		}
-	}
-}
-
-type cudTestCase struct {
-	name         string
-	line         string // if not empty, will be logged with errors, use line() to set
-	create       svcTestCase
-	beforeUpdate func(t *testing.T, storage *wrapperRESTForTests)
-	update       svcTestCase
-}
-
-func helpTestCreateUpdateDelete(t *testing.T, testCases []cudTestCase) {
-	t.Helper()
-	helpTestCreateUpdateDeleteWithFamilies(t, testCases, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol})
-}
-
-func helpTestCreateUpdateDeleteWithFamilies(t *testing.T, testCases []cudTestCase, ipFamilies []api.IPFamily) {
-	// NOTE: do not call t.Helper() here.  It's more useful for errors to be
-	// attributed to lines in this function than the caller of it.
-
-	// This test is ONLY with the gate enabled.
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-
-	storage, _, server := newStorage(t, ipFamilies)
-	defer server.Terminate(t)
-	defer storage.Store.DestroyFunc()
-
-	for _, tc := range testCases {
-		name := tc.name
-		if tc.line != "" {
-			name += "__@L" + tc.line
-		}
-		t.Run(name, func(t *testing.T) {
-			ctx := genericapirequest.NewDefaultContext()
-
-			// Create the object as specified and check the results.
-			obj, err := storage.Create(ctx, tc.create.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
-			if tc.create.expectError && err != nil {
-				return
-			}
-			if err != nil {
-				t.Fatalf("unexpected error creating service: %v", err)
-			}
-			defer storage.Delete(ctx, tc.create.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) // in case
-			if tc.create.expectError && err == nil {
-				t.Fatalf("unexpected success creating service")
-			}
-			createdSvc := obj.(*api.Service)
-			if !verifyEquiv(t, "create", &tc.create, createdSvc) {
-				return
-			}
-			verifyExpectations(t, storage, tc.create, tc.create.svc, createdSvc)
-			lastSvc := createdSvc
-
-			// The update phase is optional.
-			if tc.update.svc != nil {
-				// Allow callers to do something between create and update.
-				if tc.beforeUpdate != nil {
-					tc.beforeUpdate(t, storage)
-				}
-
-				// Update the object to the new state and check the results.
-				obj, created, err := storage.Update(ctx, tc.update.svc.Name,
-					rest.DefaultUpdatedObjectInfo(tc.update.svc), rest.ValidateAllObjectFunc,
-					rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
-				if tc.update.expectError && err != nil {
-					return
-				}
-				if err != nil {
-					t.Fatalf("unexpected error updating service: %v", err)
-				}
-				if tc.update.expectError && err == nil {
-					t.Fatalf("unexpected success updating service")
-				}
-				if created {
-					t.Fatalf("unexpected create-on-update")
-				}
-				updatedSvc := obj.(*api.Service)
-				if !verifyEquiv(t, "update", &tc.update, updatedSvc) {
-					return
-				}
-				verifyExpectations(t, storage, tc.update, createdSvc, updatedSvc)
-				lastSvc = updatedSvc
-			}
-
-			// Delete the object and check the results.
-			_, _, err = storage.Delete(ctx, tc.create.svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{})
-			if err != nil {
-				t.Fatalf("unexpected error deleting service: %v", err)
-			}
-			verifyExpectations(t, storage, svcTestCase{ /* all false */ }, lastSvc, nil)
-		})
-	}
-}
-
-type testingTInterface interface {
-	Helper()
-	Errorf(format string, args ...interface{})
-}
-
-type fakeTestingT struct {
-	t *testing.T
-}
-
-func (f fakeTestingT) Helper() {}
-
-func (f fakeTestingT) Errorf(format string, args ...interface{}) {
-	f.t.Logf(format, args...)
-}
-
-func verifyEquiv(t testingTInterface, call string, tc *svcTestCase, got *api.Service) bool {
-	t.Helper()
-
-	// For when we compare objects.
-	options := []cmp.Option{
-		// These are system-assigned values, we don't need to compare them.
-		cmpopts.IgnoreFields(api.Service{}, "UID", "ResourceVersion", "CreationTimestamp"),
-		// Treat nil slices and empty slices as the same (e.g. clusterIPs).
-		cmpopts.EquateEmpty(),
-	}
-
-	// For allocated fields, we want to be able to compare cleanly whether the
-	// input specified values or not.
-	want := tc.svc.DeepCopy()
-	if tc.expectClusterIPs || tc.expectHeadless {
-		if want.Spec.ClusterIP == "" {
-			want.Spec.ClusterIP = got.Spec.ClusterIP
-		}
-		if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-			if want.Spec.IPFamilyPolicy == nil {
-				want.Spec.IPFamilyPolicy = got.Spec.IPFamilyPolicy
-			}
-			if tc.expectStackDowngrade && len(want.Spec.ClusterIPs) > len(got.Spec.ClusterIPs) {
-				want.Spec.ClusterIPs = want.Spec.ClusterIPs[0:1]
-			} else if len(got.Spec.ClusterIPs) > len(want.Spec.ClusterIPs) {
-				want.Spec.ClusterIPs = append(want.Spec.ClusterIPs, got.Spec.ClusterIPs[len(want.Spec.ClusterIPs):]...)
-			}
-			if tc.expectStackDowngrade && len(want.Spec.IPFamilies) > len(got.Spec.ClusterIPs) {
-				want.Spec.IPFamilies = want.Spec.IPFamilies[0:1]
-			} else if len(got.Spec.IPFamilies) > len(want.Spec.IPFamilies) {
-				want.Spec.IPFamilies = append(want.Spec.IPFamilies, got.Spec.IPFamilies[len(want.Spec.IPFamilies):]...)
-			}
-		}
-	}
-
-	if tc.expectNodePorts {
-		for i := range want.Spec.Ports {
-			p := &want.Spec.Ports[i]
-			if p.NodePort == 0 {
-				p.NodePort = got.Spec.Ports[i].NodePort
-			}
-		}
-	}
-	if tc.expectHealthCheckNodePort {
-		if want.Spec.HealthCheckNodePort == 0 {
-			want.Spec.HealthCheckNodePort = got.Spec.HealthCheckNodePort
-		}
-	}
-
-	if !cmp.Equal(want, got, options...) {
-		t.Errorf("unexpected result from %s:\n%s", call, cmp.Diff(want, got, options...))
-		return false
-	}
-	return true
-}
-
-// Quis custodiet ipsos custodes?
-func TestVerifyEquiv(t *testing.T) {
-	testCases := []struct {
-		name   string
-		input  svcTestCase
-		output *api.Service
-		expect bool
-	}{{
-		name: "ExternalName",
-		input: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeExternalName),
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeExternalName),
-		expect: true,
-	}, {
-		name: "ClusterIPs_unspecified",
-		input: svcTestCase{
-			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP),
-			expectClusterIPs: true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
-		expect: true,
-	}, {
-		name: "ClusterIPs_specified",
-		input: svcTestCase{
-			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
-			expectClusterIPs: true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
-		expect: true,
-	}, {
-		name: "ClusterIPs_wrong",
-		input: svcTestCase{
-			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.0", "2000:0")),
-			expectClusterIPs: true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
-		expect: false,
-	}, {
-		name: "ClusterIPs_partial",
-		input: svcTestCase{
-			svc:              svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1")),
-			expectClusterIPs: true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeClusterIP, svctest.SetClusterIPs("10.0.0.1", "2000:1")),
-		expect: true,
-	}, {
-		name: "NodePort_unspecified",
-		input: svcTestCase{
-			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort),
-			expectClusterIPs: true,
-			expectNodePorts:  true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetUniqueNodePorts),
-		expect: true,
-	}, {
-		name: "NodePort_specified",
-		input: svcTestCase{
-			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
-			expectClusterIPs: true,
-			expectNodePorts:  true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
-		expect: true,
-	}, {
-		name: "NodePort_wrong",
-		input: svcTestCase{
-			svc:              svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(93)),
-			expectClusterIPs: true,
-			expectNodePorts:  true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeNodePort, svctest.SetNodePorts(76)),
-		expect: false,
-	}, {
-		name: "NodePort_partial",
-		input: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeNodePort,
-				svctest.SetPorts(
-					svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
-					svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
-				svctest.SetNodePorts(93)),
-			expectClusterIPs: true,
-			expectNodePorts:  true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeNodePort,
-			svctest.SetPorts(
-				svctest.MakeServicePort("p", 80, intstr.FromInt(80), api.ProtocolTCP),
-				svctest.MakeServicePort("q", 443, intstr.FromInt(443), api.ProtocolTCP)),
-			svctest.SetNodePorts(93, 76)),
-		expect: true,
-	}, {
-		name: "HealthCheckNodePort_unspecified",
-		input: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
-				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal)),
-			expectClusterIPs:          true,
-			expectNodePorts:           true,
-			expectHealthCheckNodePort: true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(93)),
-		expect: true,
-	}, {
-		name: "HealthCheckNodePort_specified",
-		input: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
-				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-				svctest.SetHealthCheckNodePort(93)),
-			expectClusterIPs:          true,
-			expectNodePorts:           true,
-			expectHealthCheckNodePort: true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(93)),
-		expect: true,
-	}, {
-		name: "HealthCheckNodePort_wrong",
-		input: svcTestCase{
-			svc: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
-				svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-				svctest.SetHealthCheckNodePort(93)),
-			expectClusterIPs:          true,
-			expectNodePorts:           true,
-			expectHealthCheckNodePort: true,
-		},
-		output: svctest.MakeService("foo", svctest.SetTypeLoadBalancer,
-			svctest.SetExternalTrafficPolicy(api.ServiceExternalTrafficPolicyTypeLocal),
-			svctest.SetHealthCheckNodePort(76)),
-		expect: false,
-	}}
-
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)()
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			result := verifyEquiv(fakeTestingT{t}, "test", &tc.input, tc.output)
-			if result != tc.expect {
-				t.Errorf("expected %v, got %v", tc.expect, result)
-			}
-		})
-	}
-}
-
-func verifyExpectations(t *testing.T, storage *wrapperRESTForTests, tc svcTestCase, before, after *api.Service) {
-	t.Helper()
-
-	if tc.expectClusterIPs {
-		proveClusterIPsAllocated(t, storage, before, after)
-	} else if tc.expectHeadless {
-		proveHeadless(t, storage, before, after)
-	} else {
-		proveClusterIPsDeallocated(t, storage, before, after)
-	}
-	if tc.expectNodePorts {
-		proveNodePortsAllocated(t, storage, before, after)
-	} else {
-		proveNodePortsDeallocated(t, storage, before, after)
-	}
-	if tc.expectHealthCheckNodePort {
-		proveHealthCheckNodePortAllocated(t, storage, before, after)
-	} else {
-		proveHealthCheckNodePortDeallocated(t, storage, before, after)
-	}
-
-	for _, p := range tc.prove {
-		p(t, storage, before, after)
-	}
-}
-
 func TestFeatureExternalName(t *testing.T) {
 	testCases := []cudTestCase{{
 		name: "valid-valid",

From f94782b4f5d5b00c8ca20d740c1fc9fc560efd86 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 21:27:15 -0700
Subject: [PATCH 57/79] Svc REST: rename allocServiceClusterIPsNew

---
 pkg/registry/core/service/storage/rest.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 13a2838e0b1..bea2d899a9a 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -77,7 +77,7 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 	//TODO(thockin): validation should not pass with empty clusterIP, but it
 	//does (and is tested!).  Fixing that all is a big PR and will have to
 	//happen later.
-	if txn, err := al.allocServiceClusterIPsNew(service, dryRun); err != nil {
+	if txn, err := al.txnAllocClusterIPs(service, dryRun); err != nil {
 		result.Revert()
 		return nil, err
 	} else {
@@ -302,8 +302,7 @@ func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service, dryRun boo
 	return allocated, err
 }
 
-//FIXME: merge into allocServiceClusterIPs rather than call it
-func (al *RESTAllocStuff) allocServiceClusterIPsNew(service *api.Service, dryRun bool) (transaction, error) {
+func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool) (transaction, error) {
 	// clusterIPs that were allocated may need to be released in case of
 	// failure at a higher level.
 	toReleaseClusterIPs, err := al.allocServiceClusterIPs(service, dryRun)

From 3b971b137c2c7ec8e999a6b277c4a4a8e07e0691 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 21:44:17 -0700
Subject: [PATCH 58/79] Svc REST: rename allocServiceNodePortsNew

---
 pkg/registry/core/service/storage/rest.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index bea2d899a9a..1aec206ab75 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -85,7 +85,7 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 	}
 
 	// Allocate ports
-	if txn, err := al.allocServiceNodePortsNew(service, dryRun); err != nil {
+	if txn, err := al.txnAllocNodePorts(service, dryRun); err != nil {
 		result.Revert()
 		return nil, err
 	} else {
@@ -956,8 +956,7 @@ func allocateHealthCheckNodePort(service *api.Service, nodePortOp *portallocator
 	return nil
 }
 
-//FIXME: rename and merge with initNodePorts
-func (al *RESTAllocStuff) allocServiceNodePortsNew(service *api.Service, dryRun bool) (transaction, error) {
+func (al *RESTAllocStuff) txnAllocNodePorts(service *api.Service, dryRun bool) (transaction, error) {
 	// The allocator tracks dry-run-ness internally.
 	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
 

From bb815e66877bc6f3456b5da3eed34fc6748625a3 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:04:11 -0700
Subject: [PATCH 59/79] Svc REST: rename allocUpdateServiceClusterIPsNew

---
 pkg/registry/core/service/storage/rest.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 1aec206ab75..5ce234dfe15 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -172,7 +172,7 @@ func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRu
 	//TODO(thockin): validation should not pass with empty clusterIP, but it
 	//does (and is tested!).  Fixing that all is a big PR and will have to
 	//happen later.
-	if txn, err := al.allocUpdateServiceClusterIPsNew(service, oldService, dryRun); err != nil {
+	if txn, err := al.txnUpdateClusterIPs(service, oldService, dryRun); err != nil {
 		result.Revert()
 		return nil, err
 	} else {
@@ -385,8 +385,7 @@ func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service, dryRun bo
 	return allocated, err
 }
 
-//FIXME: rename and merge with handleClusterIPsForUpdatedService
-func (al *RESTAllocStuff) allocUpdateServiceClusterIPsNew(service *api.Service, oldService *api.Service, dryRun bool) (transaction, error) {
+func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *api.Service, dryRun bool) (transaction, error) {
 	allocated, released, err := al.handleClusterIPsForUpdatedService(oldService, service, dryRun)
 	if err != nil {
 		return nil, err

From 7d9357b181fdaef44f80c2dcd116e23148faad86 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:04:50 -0700
Subject: [PATCH 60/79] Svc REST: rename allocUpdateServiceNodePortsNew

---
 pkg/registry/core/service/storage/rest.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 5ce234dfe15..cee583bd270 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -180,7 +180,7 @@ func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRu
 	}
 
 	// Allocate ports
-	if txn, err := al.allocUpdateServiceNodePortsNew(service, oldService, dryRun); err != nil {
+	if txn, err := al.txnUpdateNodePorts(service, oldService, dryRun); err != nil {
 		result.Revert()
 		return nil, err
 	} else {
@@ -190,8 +190,7 @@ func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRu
 	return result, nil
 }
 
-//FIXME: rename and merge with updateNodePorts?
-func (al *RESTAllocStuff) allocUpdateServiceNodePortsNew(service, oldService *api.Service, dryRun bool) (transaction, error) {
+func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, dryRun bool) (transaction, error) {
 	// The allocator tracks dry-run-ness internally.
 	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
 

From 8e330eb6119563076a44504b4d5f8e56b82cbb24 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 21:35:33 -0700
Subject: [PATCH 61/79] Svc REST: rename allocClusterIPs -> allocIPs

---
 pkg/registry/core/service/storage/rest.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index cee583bd270..d0ff5faa90f 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -231,7 +231,7 @@ func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, d
 	return txn, nil
 }
 
-func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
 	allocated := make(map[api.IPFamily]string)
 
 	for family, ip := range toAlloc {
@@ -290,7 +290,7 @@ func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service, dryRun boo
 	// get clusterIP.. empty string if user did not specify an ip
 	toAlloc[al.defaultServiceIPFamily] = service.Spec.ClusterIP
 	// alloc
-	allocated, err := al.allocClusterIPs(service, toAlloc, dryRun)
+	allocated, err := al.allocIPs(service, toAlloc, dryRun)
 
 	// set
 	if err == nil {
@@ -360,7 +360,7 @@ func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service, dryRun bo
 	}
 
 	// allocate
-	allocated, err := al.allocClusterIPs(service, toAlloc, dryRun)
+	allocated, err := al.allocIPs(service, toAlloc, dryRun)
 
 	// set if successful
 	if err == nil {
@@ -491,7 +491,7 @@ func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Serv
 		toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1]
 
 		// allocate
-		allocated, err := al.allocClusterIPs(service, toAllocate, dryRun)
+		allocated, err := al.allocIPs(service, toAllocate, dryRun)
 		// set if successful
 		if err == nil {
 			service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]]

From 1b79bbc9f448a4b5947d381de5da2b8da0cac53e Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 21:56:48 -0700
Subject: [PATCH 62/79] Svc REST: rename releaseClusterIPs -> releaseIPs

---
 pkg/registry/core/service/storage/rest.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index d0ff5faa90f..089ba29850b 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -258,7 +258,7 @@ func (al *RESTAllocStuff) allocIPs(service *api.Service, toAlloc map[api.IPFamil
 }
 
 // releases clusterIPs per family
-func (al *RESTAllocStuff) releaseClusterIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) releaseIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
 	if toRelease == nil {
 		return nil, nil
 	}
@@ -314,7 +314,7 @@ func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool)
 			if dryRun {
 				return
 			}
-			released, err := al.releaseClusterIPs(toReleaseClusterIPs)
+			released, err := al.releaseIPs(toReleaseClusterIPs)
 			if err != nil {
 				klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v",
 					service.Name, toReleaseClusterIPs, released, err)
@@ -400,7 +400,7 @@ func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *
 			if dryRun {
 				return
 			}
-			if actuallyReleased, err := al.releaseClusterIPs(released); err != nil {
+			if actuallyReleased, err := al.releaseIPs(released); err != nil {
 				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v",
 					service.Namespace, service.Name, err, released, actuallyReleased)
 			}
@@ -409,7 +409,7 @@ func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *
 			if dryRun {
 				return
 			}
-			if actuallyReleased, err := al.releaseClusterIPs(allocated); err != nil {
+			if actuallyReleased, err := al.releaseIPs(allocated); err != nil {
 				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v",
 					service.Namespace, service.Name, err, allocated, actuallyReleased)
 			}
@@ -523,7 +523,7 @@ func (al *RESTAllocStuff) releaseServiceClusterIP(service *api.Service) (release
 		toRelease[api.IPv4Protocol] = service.Spec.ClusterIP
 	}
 
-	return al.releaseClusterIPs(toRelease)
+	return al.releaseIPs(toRelease)
 }
 
 // releases allocated ClusterIPs for service that is about to be deleted
@@ -550,7 +550,7 @@ func (al *RESTAllocStuff) releaseServiceClusterIPs(service *api.Service) (releas
 			toRelease[api.IPv4Protocol] = ip
 		}
 	}
-	return al.releaseClusterIPs(toRelease)
+	return al.releaseIPs(toRelease)
 }
 
 // tests if two preferred dual-stack service have matching ClusterIPFields

From b9f1f4712a35a1273f0896a86a25540ae3343c10 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:43:26 -0700
Subject: [PATCH 63/79] Svc REST: rename allocServiceClusterIP

---
 pkg/registry/core/service/storage/rest.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 089ba29850b..140d0f91679 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -284,7 +284,7 @@ func (al *RESTAllocStuff) releaseIPs(toRelease map[api.IPFamily]string) (map[api
 
 // standard allocator for dualstackgate==Off, hard wired dependency
 // and ignores policy, families and clusterIPs
-func (al *RESTAllocStuff) allocServiceClusterIP(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocClusterIP(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
 	toAlloc := make(map[api.IPFamily]string)
 
 	// get clusterIP.. empty string if user did not specify an ip
@@ -337,7 +337,7 @@ func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service, dryRun bo
 	}
 
 	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return al.allocServiceClusterIP(service, dryRun)
+		return al.allocClusterIP(service, dryRun)
 	}
 
 	toAlloc := make(map[api.IPFamily]string)

From 2a98ec667e4367ae82fd220bb01f32bd77fbd891 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:44:10 -0700
Subject: [PATCH 64/79] Svc REST: rename allocServiceClusterIPs

---
 pkg/registry/core/service/storage/rest.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 140d0f91679..36e8d99e8d7 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -304,7 +304,7 @@ func (al *RESTAllocStuff) allocClusterIP(service *api.Service, dryRun bool) (map
 func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool) (transaction, error) {
 	// clusterIPs that were allocated may need to be released in case of
 	// failure at a higher level.
-	toReleaseClusterIPs, err := al.allocServiceClusterIPs(service, dryRun)
+	toReleaseClusterIPs, err := al.allocClusterIPs(service, dryRun)
 	if err != nil {
 		return nil, err
 	}
@@ -325,7 +325,7 @@ func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool)
 }
 
 // allocates ClusterIPs for a service
-func (al *RESTAllocStuff) allocServiceClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
+func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
 	// external name don't get ClusterIPs
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil, nil
@@ -445,7 +445,7 @@ func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Serv
 	// CASE A:
 	// Update service from ExternalName to non-ExternalName, should initialize ClusterIP.
 	if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName {
-		allocated, err := al.allocServiceClusterIPs(service, dryRun)
+		allocated, err := al.allocClusterIPs(service, dryRun)
 		return allocated, nil, err
 	}
 

From 4fb338b27988d046ae3cdd11667d669b4f532594 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:45:06 -0700
Subject: [PATCH 65/79] Svc REST: rename releaseServiceClusterIP

---
 pkg/registry/core/service/storage/rest.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 36e8d99e8d7..08f98fcfa8f 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -512,7 +512,7 @@ func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Serv
 }
 
 // for pre dual stack (gate == off). Hardwired to ClusterIP and ignores all new fields
-func (al *RESTAllocStuff) releaseServiceClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) releaseClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
 	toRelease := make(map[api.IPFamily]string)
 
 	// we need to do that to handle cases where allocator is no longer configured on
@@ -539,7 +539,7 @@ func (al *RESTAllocStuff) releaseServiceClusterIPs(service *api.Service) (releas
 	}
 
 	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return al.releaseServiceClusterIP(service)
+		return al.releaseClusterIP(service)
 	}
 
 	toRelease := make(map[api.IPFamily]string)

From 9c622230fc978921c437f4d29fca96f01f8f60dc Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:45:39 -0700
Subject: [PATCH 66/79] Svc REST: rename releaseServiceClusterIPs

---
 pkg/registry/core/service/storage/rest.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 08f98fcfa8f..50627a46176 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -96,7 +96,7 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 }
 
 func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
-	al.releaseServiceClusterIPs(svc)
+	al.releaseClusterIPs(svc)
 
 	for _, nodePort := range collectServiceNodePorts(svc) {
 		err := al.serviceNodePorts.Release(nodePort)
@@ -527,7 +527,7 @@ func (al *RESTAllocStuff) releaseClusterIP(service *api.Service) (released map[a
 }
 
 // releases allocated ClusterIPs for service that is about to be deleted
-func (al *RESTAllocStuff) releaseServiceClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) releaseClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
 	// external name don't get ClusterIPs
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil, nil

From 5dfcb905d1c358e6b187b58433879084e69773a8 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:07:39 -0700
Subject: [PATCH 67/79] Svc REST: rename handleClusterIPsForUpdatedService

---
 pkg/registry/core/service/storage/rest.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 50627a46176..28f21fe2ce8 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -385,7 +385,7 @@ func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, dryRun bool) (ma
 }
 
 func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *api.Service, dryRun bool) (transaction, error) {
-	allocated, released, err := al.handleClusterIPsForUpdatedService(oldService, service, dryRun)
+	allocated, released, err := al.updateClusterIPs(oldService, service, dryRun)
 	if err != nil {
 		return nil, err
 	}
@@ -422,7 +422,7 @@ func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *
 // this func does not perform actual release of clusterIPs. it returns
 // a map[family]ip for the caller to release when everything else has
 // executed successfully
-func (al *RESTAllocStuff) handleClusterIPsForUpdatedService(oldService *api.Service, service *api.Service, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) updateClusterIPs(oldService *api.Service, service *api.Service, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
 
 	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
 	// PreferDualStack services following a cluster change to/from

From 6b06b9bfd3fa2d97f275e3093f038a5be3a7fbf9 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:16:53 -0700
Subject: [PATCH 68/79] Svc REST: rename healthCheckNodePortUpdate

---
 pkg/registry/core/service/storage/rest.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 28f21fe2ce8..658ae121cd8 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -131,9 +131,9 @@ func shouldAllocateNodePorts(service *api.Service) bool {
 	return false
 }
 
-// healthCheckNodePortUpdate handles HealthCheckNodePort allocation/release
+// updateHealthCheckNodePort handles HealthCheckNodePort allocation/release
 // and adjusts HealthCheckNodePort during service update if needed.
-func (al *RESTAllocStuff) healthCheckNodePortUpdate(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
+func (al *RESTAllocStuff) updateHealthCheckNodePort(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
 	neededHealthCheckNodePort := apiservice.NeedsHealthCheck(oldService)
 	oldHealthCheckNodePort := oldService.Spec.HealthCheckNodePort
 
@@ -222,7 +222,7 @@ func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, d
 	}
 
 	// Handle ExternalTraffic related updates.
-	success, err := al.healthCheckNodePortUpdate(oldService, service, nodePortOp)
+	success, err := al.updateHealthCheckNodePort(oldService, service, nodePortOp)
 	if !success || err != nil {
 		txn.Revert()
 		return nil, err

From c94deffa485d360a215f7e194b49683f09ba4300 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:20:20 -0700
Subject: [PATCH 69/79] Svc REST: rename allocateHealthCheckNodePort

---
 pkg/registry/core/service/storage/rest.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index 658ae121cd8..e46a364ad5d 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -145,7 +145,7 @@ func (al *RESTAllocStuff) updateHealthCheckNodePort(oldService, service *api.Ser
 	// Insert health check node port into the service's HealthCheckNodePort field if needed.
 	case !neededHealthCheckNodePort && needsHealthCheckNodePort:
 		klog.Infof("Transition to LoadBalancer type service with ExternalTrafficPolicy=Local")
-		if err := allocateHealthCheckNodePort(service, nodePortOp); err != nil {
+		if err := allocHealthCheckNodePort(service, nodePortOp); err != nil {
 			return false, errors.NewInternalError(err)
 		}
 
@@ -931,8 +931,8 @@ func findRequestedNodePort(port int, servicePorts []api.ServicePort) int {
 	return 0
 }
 
-// allocateHealthCheckNodePort allocates health check node port to service.
-func allocateHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
+// allocHealthCheckNodePort allocates health check node port to service.
+func allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
 	healthCheckNodePort := service.Spec.HealthCheckNodePort
 	if healthCheckNodePort != 0 {
 		// If the request has a health check nodePort in mind, attempt to reserve it.
@@ -981,7 +981,7 @@ func (al *RESTAllocStuff) txnAllocNodePorts(service *api.Service, dryRun bool) (
 
 	// Handle ExternalTraffic related fields during service creation.
 	if apiservice.NeedsHealthCheck(service) {
-		if err := allocateHealthCheckNodePort(service, nodePortOp); err != nil {
+		if err := allocHealthCheckNodePort(service, nodePortOp); err != nil {
 			txn.Revert()
 			return nil, errors.NewInternalError(err)
 		}

From 1ce9807de0b5c5e9e65ecc9e5dcb4f72b44e12a7 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:49:53 -0700
Subject: [PATCH 70/79] Svc REST: Make allocHCNP a method: more consistent

---
 pkg/registry/core/service/storage/rest.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go
index e46a364ad5d..5b34692a6ca 100644
--- a/pkg/registry/core/service/storage/rest.go
+++ b/pkg/registry/core/service/storage/rest.go
@@ -145,7 +145,7 @@ func (al *RESTAllocStuff) updateHealthCheckNodePort(oldService, service *api.Ser
 	// Insert health check node port into the service's HealthCheckNodePort field if needed.
 	case !neededHealthCheckNodePort && needsHealthCheckNodePort:
 		klog.Infof("Transition to LoadBalancer type service with ExternalTrafficPolicy=Local")
-		if err := allocHealthCheckNodePort(service, nodePortOp); err != nil {
+		if err := al.allocHealthCheckNodePort(service, nodePortOp); err != nil {
 			return false, errors.NewInternalError(err)
 		}
 
@@ -932,7 +932,7 @@ func findRequestedNodePort(port int, servicePorts []api.ServicePort) int {
 }
 
 // allocHealthCheckNodePort allocates health check node port to service.
-func allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
+func (al *RESTAllocStuff) allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
 	healthCheckNodePort := service.Spec.HealthCheckNodePort
 	if healthCheckNodePort != 0 {
 		// If the request has a health check nodePort in mind, attempt to reserve it.
@@ -981,7 +981,7 @@ func (al *RESTAllocStuff) txnAllocNodePorts(service *api.Service, dryRun bool) (
 
 	// Handle ExternalTraffic related fields during service creation.
 	if apiservice.NeedsHealthCheck(service) {
-		if err := allocHealthCheckNodePort(service, nodePortOp); err != nil {
+		if err := al.allocHealthCheckNodePort(service, nodePortOp); err != nil {
 			txn.Revert()
 			return nil, errors.NewInternalError(err)
 		}

From 7ce34e311aee06036b4f39951cd98e9bae44f05c Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 22:56:32 -0700
Subject: [PATCH 71/79] Svc REST: rename rest.go -> alloc.go

---
 pkg/registry/core/service/storage/{rest.go => alloc.go} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename pkg/registry/core/service/storage/{rest.go => alloc.go} (100%)

diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/alloc.go
similarity index 100%
rename from pkg/registry/core/service/storage/rest.go
rename to pkg/registry/core/service/storage/alloc.go

From 4c9bc5a53c62059067efb4b985f1c73175a874e4 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 23:18:31 -0700
Subject: [PATCH 72/79] Svc REST: Move alloc code around

This is detrimental to future `blame` but makes it so much morereadable
I convinced myself it was worthwhile.
---
 pkg/registry/core/service/storage/alloc.go | 1269 ++++++++++----------
 1 file changed, 634 insertions(+), 635 deletions(-)

diff --git a/pkg/registry/core/service/storage/alloc.go b/pkg/registry/core/service/storage/alloc.go
index 5b34692a6ca..4969143b932 100644
--- a/pkg/registry/core/service/storage/alloc.go
+++ b/pkg/registry/core/service/storage/alloc.go
@@ -95,573 +95,6 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 	return result, nil
 }
 
-func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
-	al.releaseClusterIPs(svc)
-
-	for _, nodePort := range collectServiceNodePorts(svc) {
-		err := al.serviceNodePorts.Release(nodePort)
-		if err != nil {
-			// these should be caught by an eventual reconciliation / restart
-			utilruntime.HandleError(fmt.Errorf("Error releasing service %s node port %d: %v", svc.Name, nodePort, err))
-		}
-	}
-
-	if apiservice.NeedsHealthCheck(svc) {
-		nodePort := svc.Spec.HealthCheckNodePort
-		if nodePort > 0 {
-			err := al.serviceNodePorts.Release(int(nodePort))
-			if err != nil {
-				// these should be caught by an eventual reconciliation / restart
-				utilruntime.HandleError(fmt.Errorf("Error releasing service %s health check node port %d: %v", svc.Name, nodePort, err))
-			}
-		}
-	}
-}
-
-func shouldAllocateNodePorts(service *api.Service) bool {
-	if service.Spec.Type == api.ServiceTypeNodePort {
-		return true
-	}
-	if service.Spec.Type == api.ServiceTypeLoadBalancer {
-		if utilfeature.DefaultFeatureGate.Enabled(features.ServiceLBNodePortControl) {
-			return *service.Spec.AllocateLoadBalancerNodePorts
-		}
-		return true
-	}
-	return false
-}
-
-// updateHealthCheckNodePort handles HealthCheckNodePort allocation/release
-// and adjusts HealthCheckNodePort during service update if needed.
-func (al *RESTAllocStuff) updateHealthCheckNodePort(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
-	neededHealthCheckNodePort := apiservice.NeedsHealthCheck(oldService)
-	oldHealthCheckNodePort := oldService.Spec.HealthCheckNodePort
-
-	needsHealthCheckNodePort := apiservice.NeedsHealthCheck(service)
-
-	switch {
-	// Case 1: Transition from don't need HealthCheckNodePort to needs HealthCheckNodePort.
-	// Allocate a health check node port or attempt to reserve the user-specified one if provided.
-	// Insert health check node port into the service's HealthCheckNodePort field if needed.
-	case !neededHealthCheckNodePort && needsHealthCheckNodePort:
-		klog.Infof("Transition to LoadBalancer type service with ExternalTrafficPolicy=Local")
-		if err := al.allocHealthCheckNodePort(service, nodePortOp); err != nil {
-			return false, errors.NewInternalError(err)
-		}
-
-	// Case 2: Transition from needs HealthCheckNodePort to don't need HealthCheckNodePort.
-	// Free the existing healthCheckNodePort and clear the HealthCheckNodePort field.
-	case neededHealthCheckNodePort && !needsHealthCheckNodePort:
-		klog.Infof("Transition to non LoadBalancer type service or LoadBalancer type service with ExternalTrafficPolicy=Global")
-		klog.V(4).Infof("Releasing healthCheckNodePort: %d", oldHealthCheckNodePort)
-		nodePortOp.ReleaseDeferred(int(oldHealthCheckNodePort))
-	}
-	return true, nil
-}
-
-func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRun bool) (transaction, error) {
-	result := metaTransaction{}
-
-	// Ensure IP family fields are correctly initialized.  We do it here, since
-	// we want this to be visible even when dryRun == true.
-	if err := al.initIPFamilyFields(oldService, service); err != nil {
-		return nil, err
-	}
-
-	// Allocate ClusterIPs
-	//TODO(thockin): validation should not pass with empty clusterIP, but it
-	//does (and is tested!).  Fixing that all is a big PR and will have to
-	//happen later.
-	if txn, err := al.txnUpdateClusterIPs(service, oldService, dryRun); err != nil {
-		result.Revert()
-		return nil, err
-	} else {
-		result = append(result, txn)
-	}
-
-	// Allocate ports
-	if txn, err := al.txnUpdateNodePorts(service, oldService, dryRun); err != nil {
-		result.Revert()
-		return nil, err
-	} else {
-		result = append(result, txn)
-	}
-
-	return result, nil
-}
-
-func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, dryRun bool) (transaction, error) {
-	// The allocator tracks dry-run-ness internally.
-	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
-
-	txn := callbackTransaction{
-		commit: func() {
-			nodePortOp.Commit()
-			// We don't NEED to call Finish() here, but for that package says
-			// to, so for future-safety, we will.
-			nodePortOp.Finish()
-		},
-		revert: func() {
-			// Weirdly named but this will revert if commit wasn't called
-			nodePortOp.Finish()
-		},
-	}
-
-	// Update service from NodePort or LoadBalancer to ExternalName or ClusterIP, should release NodePort if exists.
-	if (oldService.Spec.Type == api.ServiceTypeNodePort || oldService.Spec.Type == api.ServiceTypeLoadBalancer) &&
-		(service.Spec.Type == api.ServiceTypeExternalName || service.Spec.Type == api.ServiceTypeClusterIP) {
-		releaseNodePorts(oldService, nodePortOp)
-	}
-
-	// Update service from any type to NodePort or LoadBalancer, should update NodePort.
-	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
-		if err := updateNodePorts(oldService, service, nodePortOp); err != nil {
-			txn.Revert()
-			return nil, err
-		}
-	}
-
-	// Handle ExternalTraffic related updates.
-	success, err := al.updateHealthCheckNodePort(oldService, service, nodePortOp)
-	if !success || err != nil {
-		txn.Revert()
-		return nil, err
-	}
-
-	return txn, nil
-}
-
-func (al *RESTAllocStuff) allocIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
-	allocated := make(map[api.IPFamily]string)
-
-	for family, ip := range toAlloc {
-		allocator := al.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
-		if dryRun {
-			allocator = allocator.DryRun()
-		}
-		if ip == "" {
-			allocatedIP, err := allocator.AllocateNext()
-			if err != nil {
-				return allocated, errors.NewInternalError(fmt.Errorf("failed to allocate a serviceIP: %v", err))
-			}
-			allocated[family] = allocatedIP.String()
-		} else {
-			parsedIP := netutils.ParseIPSloppy(ip)
-			if err := allocator.Allocate(parsedIP); err != nil {
-				el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs"), service.Spec.ClusterIPs, fmt.Sprintf("failed to allocate IP %v: %v", ip, err))}
-				return allocated, errors.NewInvalid(api.Kind("Service"), service.Name, el)
-			}
-			allocated[family] = ip
-		}
-	}
-	return allocated, nil
-}
-
-// releases clusterIPs per family
-func (al *RESTAllocStuff) releaseIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
-	if toRelease == nil {
-		return nil, nil
-	}
-
-	released := make(map[api.IPFamily]string)
-	for family, ip := range toRelease {
-		allocator, ok := al.serviceIPAllocatorsByFamily[family]
-		if !ok {
-			// cluster was configured for dual stack, then single stack
-			klog.V(4).Infof("delete service. Not releasing ClusterIP:%v because IPFamily:%v is no longer configured on server", ip, family)
-			continue
-		}
-
-		parsedIP := netutils.ParseIPSloppy(ip)
-		if err := allocator.Release(parsedIP); err != nil {
-			return released, err
-		}
-		released[family] = ip
-	}
-
-	return released, nil
-}
-
-// standard allocator for dualstackgate==Off, hard wired dependency
-// and ignores policy, families and clusterIPs
-func (al *RESTAllocStuff) allocClusterIP(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
-	toAlloc := make(map[api.IPFamily]string)
-
-	// get clusterIP.. empty string if user did not specify an ip
-	toAlloc[al.defaultServiceIPFamily] = service.Spec.ClusterIP
-	// alloc
-	allocated, err := al.allocIPs(service, toAlloc, dryRun)
-
-	// set
-	if err == nil {
-		service.Spec.ClusterIP = allocated[al.defaultServiceIPFamily]
-		service.Spec.ClusterIPs = []string{allocated[al.defaultServiceIPFamily]}
-	}
-
-	return allocated, err
-}
-
-func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool) (transaction, error) {
-	// clusterIPs that were allocated may need to be released in case of
-	// failure at a higher level.
-	toReleaseClusterIPs, err := al.allocClusterIPs(service, dryRun)
-	if err != nil {
-		return nil, err
-	}
-
-	txn := callbackTransaction{
-		revert: func() {
-			if dryRun {
-				return
-			}
-			released, err := al.releaseIPs(toReleaseClusterIPs)
-			if err != nil {
-				klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v",
-					service.Name, toReleaseClusterIPs, released, err)
-			}
-		},
-	}
-	return txn, nil
-}
-
-// allocates ClusterIPs for a service
-func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
-	// external name don't get ClusterIPs
-	if service.Spec.Type == api.ServiceTypeExternalName {
-		return nil, nil
-	}
-
-	// headless don't get ClusterIPs
-	if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone {
-		return nil, nil
-	}
-
-	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return al.allocClusterIP(service, dryRun)
-	}
-
-	toAlloc := make(map[api.IPFamily]string)
-	// at this stage, the only fact we know is that service has correct ip families
-	// assigned to it. It may have partial assigned ClusterIPs (Upgrade to dual stack)
-	// may have no ips at all. The below loop is meant to fix this
-	// (we also know that this cluster has these families)
-
-	// if there is no slice to work with
-	if service.Spec.ClusterIPs == nil {
-		service.Spec.ClusterIPs = make([]string, 0, len(service.Spec.IPFamilies))
-	}
-
-	for i, ipFamily := range service.Spec.IPFamilies {
-		if i > (len(service.Spec.ClusterIPs) - 1) {
-			service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* just a marker */)
-		}
-
-		toAlloc[ipFamily] = service.Spec.ClusterIPs[i]
-	}
-
-	// allocate
-	allocated, err := al.allocIPs(service, toAlloc, dryRun)
-
-	// set if successful
-	if err == nil {
-		for family, ip := range allocated {
-			for i, check := range service.Spec.IPFamilies {
-				if family == check {
-					service.Spec.ClusterIPs[i] = ip
-					// while we technically don't need to do that testing rest does not
-					// go through conversion logic but goes through validation *sigh*.
-					// so we set ClusterIP here as well
-					// because the testing code expects valid (as they are output-ed from conversion)
-					// as it patches fields
-					if i == 0 {
-						service.Spec.ClusterIP = ip
-					}
-				}
-			}
-		}
-	}
-
-	return allocated, err
-}
-
-func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *api.Service, dryRun bool) (transaction, error) {
-	allocated, released, err := al.updateClusterIPs(oldService, service, dryRun)
-	if err != nil {
-		return nil, err
-	}
-
-	// on failure: Any newly allocated IP must be released back
-	// on failure: Any previously allocated IP that would have been released,
-	//             must *not* be released
-	// on success: Any previously allocated IP that should be released, will be
-	//             released
-	txn := callbackTransaction{
-		commit: func() {
-			if dryRun {
-				return
-			}
-			if actuallyReleased, err := al.releaseIPs(released); err != nil {
-				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v",
-					service.Namespace, service.Name, err, released, actuallyReleased)
-			}
-		},
-		revert: func() {
-			if dryRun {
-				return
-			}
-			if actuallyReleased, err := al.releaseIPs(allocated); err != nil {
-				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v",
-					service.Namespace, service.Name, err, allocated, actuallyReleased)
-			}
-		},
-	}
-	return txn, nil
-}
-
-// handles type change/upgrade/downgrade change type for an update service
-// this func does not perform actual release of clusterIPs. it returns
-// a map[family]ip for the caller to release when everything else has
-// executed successfully
-func (al *RESTAllocStuff) updateClusterIPs(oldService *api.Service, service *api.Service, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
-
-	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
-	// PreferDualStack services following a cluster change to/from
-	// dual-stackness.
-	//
-	// That means a PreferDualStack service will only be upgraded/downgraded
-	// when:
-	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
-	// - adding or removing a secondary clusterIP or ipFamily
-	if isMatchingPreferDualStackClusterIPFields(oldService, service) {
-		return allocated, toRelease, nil // nothing more to do.
-	}
-
-	// use cases:
-	// A: service changing types from ExternalName TO ClusterIP types ==> allocate all new
-	// B: service changing types from ClusterIP types TO ExternalName ==> release all allocated
-	// C: Service upgrading to dual stack  ==> partial allocation
-	// D: service downgrading from dual stack ==> partial release
-
-	// CASE A:
-	// Update service from ExternalName to non-ExternalName, should initialize ClusterIP.
-	if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName {
-		allocated, err := al.allocClusterIPs(service, dryRun)
-		return allocated, nil, err
-	}
-
-	// CASE B:
-
-	// if headless service then we bail out early (no clusterIPs management needed)
-	if len(oldService.Spec.ClusterIPs) > 0 && oldService.Spec.ClusterIPs[0] == api.ClusterIPNone {
-		return nil, nil, nil
-	}
-
-	// Update service from non-ExternalName to ExternalName, should release ClusterIP if exists.
-	if oldService.Spec.Type != api.ServiceTypeExternalName && service.Spec.Type == api.ServiceTypeExternalName {
-		toRelease = make(map[api.IPFamily]string)
-		if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-			// for non dual stack enabled cluster we use clusterIPs
-			toRelease[al.defaultServiceIPFamily] = oldService.Spec.ClusterIP
-		} else {
-			// dual stack is enabled, collect ClusterIPs by families
-			for i, family := range oldService.Spec.IPFamilies {
-				toRelease[family] = oldService.Spec.ClusterIPs[i]
-			}
-		}
-
-		return nil, toRelease, nil
-	}
-
-	// upgrade and downgrade are specific to dualstack
-	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return nil, nil, nil
-	}
-
-	upgraded := len(oldService.Spec.IPFamilies) == 1 && len(service.Spec.IPFamilies) == 2
-	downgraded := len(oldService.Spec.IPFamilies) == 2 && len(service.Spec.IPFamilies) == 1
-
-	// CASE C:
-	if upgraded {
-		toAllocate := make(map[api.IPFamily]string)
-		// if secondary ip was named, just get it. if not add a marker
-		if len(service.Spec.ClusterIPs) < 2 {
-			service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* marker */)
-		}
-
-		toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1]
-
-		// allocate
-		allocated, err := al.allocIPs(service, toAllocate, dryRun)
-		// set if successful
-		if err == nil {
-			service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]]
-		}
-
-		return allocated, nil, err
-	}
-
-	// CASE D:
-	if downgraded {
-		toRelease = make(map[api.IPFamily]string)
-		toRelease[oldService.Spec.IPFamilies[1]] = oldService.Spec.ClusterIPs[1]
-		// note: we don't release clusterIP, this is left to clean up in the action itself
-		return nil, toRelease, err
-	}
-	// it was not an upgrade nor downgrade
-	return nil, nil, nil
-}
-
-// for pre dual stack (gate == off). Hardwired to ClusterIP and ignores all new fields
-func (al *RESTAllocStuff) releaseClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
-	toRelease := make(map[api.IPFamily]string)
-
-	// we need to do that to handle cases where allocator is no longer configured on
-	// cluster
-	if netutils.IsIPv6String(service.Spec.ClusterIP) {
-		toRelease[api.IPv6Protocol] = service.Spec.ClusterIP
-	} else {
-		toRelease[api.IPv4Protocol] = service.Spec.ClusterIP
-	}
-
-	return al.releaseIPs(toRelease)
-}
-
-// releases allocated ClusterIPs for service that is about to be deleted
-func (al *RESTAllocStuff) releaseClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
-	// external name don't get ClusterIPs
-	if service.Spec.Type == api.ServiceTypeExternalName {
-		return nil, nil
-	}
-
-	// headless don't get ClusterIPs
-	if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone {
-		return nil, nil
-	}
-
-	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
-		return al.releaseClusterIP(service)
-	}
-
-	toRelease := make(map[api.IPFamily]string)
-	for _, ip := range service.Spec.ClusterIPs {
-		if netutils.IsIPv6String(ip) {
-			toRelease[api.IPv6Protocol] = ip
-		} else {
-			toRelease[api.IPv4Protocol] = ip
-		}
-	}
-	return al.releaseIPs(toRelease)
-}
-
-// tests if two preferred dual-stack service have matching ClusterIPFields
-// assumption: old service is a valid, default service (e.g., loaded from store)
-func isMatchingPreferDualStackClusterIPFields(oldService, service *api.Service) bool {
-	if oldService == nil {
-		return false
-	}
-
-	if service.Spec.IPFamilyPolicy == nil {
-		return false
-	}
-
-	// if type mutated then it is an update
-	// that needs to run through the entire process.
-	if oldService.Spec.Type != service.Spec.Type {
-		return false
-	}
-	// both must be type that gets an IP assigned
-	if service.Spec.Type != api.ServiceTypeClusterIP &&
-		service.Spec.Type != api.ServiceTypeNodePort &&
-		service.Spec.Type != api.ServiceTypeLoadBalancer {
-		return false
-	}
-
-	// both must be of IPFamilyPolicy==PreferDualStack
-	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicyPreferDualStack {
-		return false
-	}
-
-	if oldService.Spec.IPFamilyPolicy != nil && *(oldService.Spec.IPFamilyPolicy) != api.IPFamilyPolicyPreferDualStack {
-		return false
-	}
-
-	if !sameClusterIPs(oldService, service) {
-		return false
-	}
-
-	if !sameIPFamilies(oldService, service) {
-		return false
-	}
-
-	// they match on
-	// Policy: preferDualStack
-	// ClusterIPs
-	// IPFamilies
-	return true
-}
-
-func sameClusterIPs(lhs, rhs *api.Service) bool {
-	if len(rhs.Spec.ClusterIPs) != len(lhs.Spec.ClusterIPs) {
-		return false
-	}
-
-	for i, ip := range rhs.Spec.ClusterIPs {
-		if lhs.Spec.ClusterIPs[i] != ip {
-			return false
-		}
-	}
-
-	return true
-}
-
-func reducedClusterIPs(before, after *api.Service) bool {
-	if len(after.Spec.ClusterIPs) == 0 { // Not specified
-		return false
-	}
-	return len(after.Spec.ClusterIPs) < len(before.Spec.ClusterIPs)
-}
-
-func sameIPFamilies(lhs, rhs *api.Service) bool {
-	if len(rhs.Spec.IPFamilies) != len(lhs.Spec.IPFamilies) {
-		return false
-	}
-
-	for i, family := range rhs.Spec.IPFamilies {
-		if lhs.Spec.IPFamilies[i] != family {
-			return false
-		}
-	}
-
-	return true
-}
-
-func reducedIPFamilies(before, after *api.Service) bool {
-	if len(after.Spec.IPFamilies) == 0 { // Not specified
-		return false
-	}
-	return len(after.Spec.IPFamilies) < len(before.Spec.IPFamilies)
-}
-
-// Helper to get the IP family of a given IP.
-func familyOf(ip string) api.IPFamily {
-	if netutils.IsIPv4String(ip) {
-		return api.IPv4Protocol
-	}
-	if netutils.IsIPv6String(ip) {
-		return api.IPv6Protocol
-	}
-	return api.IPFamily("unknown")
-}
-
-// Helper to avoid nil-checks all over.  Callers of this need to be checking
-// for an exact value.
-func getIPFamilyPolicy(svc *api.Service) api.IPFamilyPolicyType {
-	if svc.Spec.IPFamilyPolicy == nil {
-		return "" // callers need to handle this
-	}
-	return *svc.Spec.IPFamilyPolicy
-}
-
 // attempts to default service ip families according to cluster configuration
 // while ensuring that provided families are configured on cluster.
 func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) error {
@@ -871,87 +304,157 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 	return nil
 }
 
-func isValidAddress(ctx context.Context, addr *api.EndpointAddress, pods rest.Getter) error {
-	if addr.TargetRef == nil {
-		return fmt.Errorf("Address has no target ref, skipping: %v", addr)
-	}
-	if genericapirequest.NamespaceValue(ctx) != addr.TargetRef.Namespace {
-		return fmt.Errorf("Address namespace doesn't match context namespace")
-	}
-	obj, err := pods.Get(ctx, addr.TargetRef.Name, &metav1.GetOptions{})
+func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool) (transaction, error) {
+	// clusterIPs that were allocated may need to be released in case of
+	// failure at a higher level.
+	toReleaseClusterIPs, err := al.allocClusterIPs(service, dryRun)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	pod, ok := obj.(*api.Pod)
-	if !ok {
-		return fmt.Errorf("failed to cast to pod: %v", obj)
+
+	txn := callbackTransaction{
+		revert: func() {
+			if dryRun {
+				return
+			}
+			released, err := al.releaseIPs(toReleaseClusterIPs)
+			if err != nil {
+				klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v",
+					service.Name, toReleaseClusterIPs, released, err)
+			}
+		},
 	}
-	if pod == nil {
-		return fmt.Errorf("pod is missing, skipping (%s/%s)", addr.TargetRef.Namespace, addr.TargetRef.Name)
-	}
-	for _, podIP := range pod.Status.PodIPs {
-		if podIP.IP == addr.IP {
-			return nil
-		}
-	}
-	return fmt.Errorf("pod ip(s) doesn't match endpoint ip, skipping: %v vs %s (%s/%s)", pod.Status.PodIPs, addr.IP, addr.TargetRef.Namespace, addr.TargetRef.Name)
+	return txn, nil
 }
 
-// This is O(N), but we expect haystack to be small;
-// so small that we expect a linear search to be faster
-func containsNumber(haystack []int, needle int) bool {
-	for _, v := range haystack {
-		if v == needle {
-			return true
+// allocates ClusterIPs for a service
+func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
+	// external name don't get ClusterIPs
+	if service.Spec.Type == api.ServiceTypeExternalName {
+		return nil, nil
+	}
+
+	// headless don't get ClusterIPs
+	if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone {
+		return nil, nil
+	}
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		return al.allocClusterIP(service, dryRun)
+	}
+
+	toAlloc := make(map[api.IPFamily]string)
+	// at this stage, the only fact we know is that service has correct ip families
+	// assigned to it. It may have partial assigned ClusterIPs (Upgrade to dual stack)
+	// may have no ips at all. The below loop is meant to fix this
+	// (we also know that this cluster has these families)
+
+	// if there is no slice to work with
+	if service.Spec.ClusterIPs == nil {
+		service.Spec.ClusterIPs = make([]string, 0, len(service.Spec.IPFamilies))
+	}
+
+	for i, ipFamily := range service.Spec.IPFamilies {
+		if i > (len(service.Spec.ClusterIPs) - 1) {
+			service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* just a marker */)
+		}
+
+		toAlloc[ipFamily] = service.Spec.ClusterIPs[i]
+	}
+
+	// allocate
+	allocated, err := al.allocIPs(service, toAlloc, dryRun)
+
+	// set if successful
+	if err == nil {
+		for family, ip := range allocated {
+			for i, check := range service.Spec.IPFamilies {
+				if family == check {
+					service.Spec.ClusterIPs[i] = ip
+					// while we technically don't need to do that testing rest does not
+					// go through conversion logic but goes through validation *sigh*.
+					// so we set ClusterIP here as well
+					// because the testing code expects valid (as they are output-ed from conversion)
+					// as it patches fields
+					if i == 0 {
+						service.Spec.ClusterIP = ip
+					}
+				}
+			}
 		}
 	}
-	return false
+
+	return allocated, err
 }
 
-// This is O(N), but we expect serviceNodePorts to be small;
-// so small that we expect a linear search to be faster
-func containsNodePort(serviceNodePorts []ServiceNodePort, serviceNodePort ServiceNodePort) bool {
-	for _, snp := range serviceNodePorts {
-		if snp == serviceNodePort {
-			return true
-		}
+// standard allocator for dualstackgate==Off, hard wired dependency
+// and ignores policy, families and clusterIPs
+func (al *RESTAllocStuff) allocClusterIP(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
+	toAlloc := make(map[api.IPFamily]string)
+
+	// get clusterIP.. empty string if user did not specify an ip
+	toAlloc[al.defaultServiceIPFamily] = service.Spec.ClusterIP
+	// alloc
+	allocated, err := al.allocIPs(service, toAlloc, dryRun)
+
+	// set
+	if err == nil {
+		service.Spec.ClusterIP = allocated[al.defaultServiceIPFamily]
+		service.Spec.ClusterIPs = []string{allocated[al.defaultServiceIPFamily]}
 	}
-	return false
+
+	return allocated, err
 }
 
-// Loop through the service ports list, find one with the same port number and
-// NodePort specified, return this NodePort otherwise return 0.
-func findRequestedNodePort(port int, servicePorts []api.ServicePort) int {
-	for i := range servicePorts {
-		servicePort := servicePorts[i]
-		if port == int(servicePort.Port) && servicePort.NodePort != 0 {
-			return int(servicePort.NodePort)
+func (al *RESTAllocStuff) allocIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
+	allocated := make(map[api.IPFamily]string)
+
+	for family, ip := range toAlloc {
+		allocator := al.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
+		if dryRun {
+			allocator = allocator.DryRun()
+		}
+		if ip == "" {
+			allocatedIP, err := allocator.AllocateNext()
+			if err != nil {
+				return allocated, errors.NewInternalError(fmt.Errorf("failed to allocate a serviceIP: %v", err))
+			}
+			allocated[family] = allocatedIP.String()
+		} else {
+			parsedIP := netutils.ParseIPSloppy(ip)
+			if err := allocator.Allocate(parsedIP); err != nil {
+				el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs"), service.Spec.ClusterIPs, fmt.Sprintf("failed to allocate IP %v: %v", ip, err))}
+				return allocated, errors.NewInvalid(api.Kind("Service"), service.Name, el)
+			}
+			allocated[family] = ip
 		}
 	}
-	return 0
+	return allocated, nil
 }
 
-// allocHealthCheckNodePort allocates health check node port to service.
-func (al *RESTAllocStuff) allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
-	healthCheckNodePort := service.Spec.HealthCheckNodePort
-	if healthCheckNodePort != 0 {
-		// If the request has a health check nodePort in mind, attempt to reserve it.
-		err := nodePortOp.Allocate(int(healthCheckNodePort))
-		if err != nil {
-			return fmt.Errorf("failed to allocate requested HealthCheck NodePort %v: %v",
-				healthCheckNodePort, err)
-		}
-		klog.V(4).Infof("Reserved user requested healthCheckNodePort: %d", healthCheckNodePort)
-	} else {
-		// If the request has no health check nodePort specified, allocate any.
-		healthCheckNodePort, err := nodePortOp.AllocateNext()
-		if err != nil {
-			return fmt.Errorf("failed to allocate a HealthCheck NodePort %v: %v", healthCheckNodePort, err)
-		}
-		service.Spec.HealthCheckNodePort = int32(healthCheckNodePort)
-		klog.V(4).Infof("Reserved allocated healthCheckNodePort: %d", healthCheckNodePort)
+// releases clusterIPs per family
+func (al *RESTAllocStuff) releaseIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
+	if toRelease == nil {
+		return nil, nil
 	}
-	return nil
+
+	released := make(map[api.IPFamily]string)
+	for family, ip := range toRelease {
+		allocator, ok := al.serviceIPAllocatorsByFamily[family]
+		if !ok {
+			// cluster was configured for dual stack, then single stack
+			klog.V(4).Infof("delete service. Not releasing ClusterIP:%v because IPFamily:%v is no longer configured on server", ip, family)
+			continue
+		}
+
+		parsedIP := netutils.ParseIPSloppy(ip)
+		if err := allocator.Release(parsedIP); err != nil {
+			return released, err
+		}
+		released[family] = ip
+	}
+
+	return released, nil
 }
 
 func (al *RESTAllocStuff) txnAllocNodePorts(service *api.Service, dryRun bool) (transaction, error) {
@@ -1043,6 +546,235 @@ func initNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocatio
 	return nil
 }
 
+// allocHealthCheckNodePort allocates health check node port to service.
+func (al *RESTAllocStuff) allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
+	healthCheckNodePort := service.Spec.HealthCheckNodePort
+	if healthCheckNodePort != 0 {
+		// If the request has a health check nodePort in mind, attempt to reserve it.
+		err := nodePortOp.Allocate(int(healthCheckNodePort))
+		if err != nil {
+			return fmt.Errorf("failed to allocate requested HealthCheck NodePort %v: %v",
+				healthCheckNodePort, err)
+		}
+		klog.V(4).Infof("Reserved user requested healthCheckNodePort: %d", healthCheckNodePort)
+	} else {
+		// If the request has no health check nodePort specified, allocate any.
+		healthCheckNodePort, err := nodePortOp.AllocateNext()
+		if err != nil {
+			return fmt.Errorf("failed to allocate a HealthCheck NodePort %v: %v", healthCheckNodePort, err)
+		}
+		service.Spec.HealthCheckNodePort = int32(healthCheckNodePort)
+		klog.V(4).Infof("Reserved allocated healthCheckNodePort: %d", healthCheckNodePort)
+	}
+	return nil
+}
+
+func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRun bool) (transaction, error) {
+	result := metaTransaction{}
+
+	// Ensure IP family fields are correctly initialized.  We do it here, since
+	// we want this to be visible even when dryRun == true.
+	if err := al.initIPFamilyFields(oldService, service); err != nil {
+		return nil, err
+	}
+
+	// Allocate ClusterIPs
+	//TODO(thockin): validation should not pass with empty clusterIP, but it
+	//does (and is tested!).  Fixing that all is a big PR and will have to
+	//happen later.
+	if txn, err := al.txnUpdateClusterIPs(service, oldService, dryRun); err != nil {
+		result.Revert()
+		return nil, err
+	} else {
+		result = append(result, txn)
+	}
+
+	// Allocate ports
+	if txn, err := al.txnUpdateNodePorts(service, oldService, dryRun); err != nil {
+		result.Revert()
+		return nil, err
+	} else {
+		result = append(result, txn)
+	}
+
+	return result, nil
+}
+
+func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *api.Service, dryRun bool) (transaction, error) {
+	allocated, released, err := al.updateClusterIPs(oldService, service, dryRun)
+	if err != nil {
+		return nil, err
+	}
+
+	// on failure: Any newly allocated IP must be released back
+	// on failure: Any previously allocated IP that would have been released,
+	//             must *not* be released
+	// on success: Any previously allocated IP that should be released, will be
+	//             released
+	txn := callbackTransaction{
+		commit: func() {
+			if dryRun {
+				return
+			}
+			if actuallyReleased, err := al.releaseIPs(released); err != nil {
+				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v",
+					service.Namespace, service.Name, err, released, actuallyReleased)
+			}
+		},
+		revert: func() {
+			if dryRun {
+				return
+			}
+			if actuallyReleased, err := al.releaseIPs(allocated); err != nil {
+				klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v",
+					service.Namespace, service.Name, err, allocated, actuallyReleased)
+			}
+		},
+	}
+	return txn, nil
+}
+
+// handles type change/upgrade/downgrade change type for an update service
+// this func does not perform actual release of clusterIPs. it returns
+// a map[family]ip for the caller to release when everything else has
+// executed successfully
+func (al *RESTAllocStuff) updateClusterIPs(oldService *api.Service, service *api.Service, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
+
+	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
+	// PreferDualStack services following a cluster change to/from
+	// dual-stackness.
+	//
+	// That means a PreferDualStack service will only be upgraded/downgraded
+	// when:
+	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
+	// - adding or removing a secondary clusterIP or ipFamily
+	if isMatchingPreferDualStackClusterIPFields(oldService, service) {
+		return allocated, toRelease, nil // nothing more to do.
+	}
+
+	// use cases:
+	// A: service changing types from ExternalName TO ClusterIP types ==> allocate all new
+	// B: service changing types from ClusterIP types TO ExternalName ==> release all allocated
+	// C: Service upgrading to dual stack  ==> partial allocation
+	// D: service downgrading from dual stack ==> partial release
+
+	// CASE A:
+	// Update service from ExternalName to non-ExternalName, should initialize ClusterIP.
+	if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName {
+		allocated, err := al.allocClusterIPs(service, dryRun)
+		return allocated, nil, err
+	}
+
+	// if headless service then we bail out early (no clusterIPs management needed)
+	if len(oldService.Spec.ClusterIPs) > 0 && oldService.Spec.ClusterIPs[0] == api.ClusterIPNone {
+		return nil, nil, nil
+	}
+
+	// CASE B:
+	// Update service from non-ExternalName to ExternalName, should release ClusterIP if exists.
+	if oldService.Spec.Type != api.ServiceTypeExternalName && service.Spec.Type == api.ServiceTypeExternalName {
+		toRelease = make(map[api.IPFamily]string)
+		if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+			// for non dual stack enabled cluster we use clusterIPs
+			toRelease[al.defaultServiceIPFamily] = oldService.Spec.ClusterIP
+		} else {
+			// dual stack is enabled, collect ClusterIPs by families
+			for i, family := range oldService.Spec.IPFamilies {
+				toRelease[family] = oldService.Spec.ClusterIPs[i]
+			}
+		}
+
+		return nil, toRelease, nil
+	}
+
+	// upgrade and downgrade are specific to dualstack
+	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		return nil, nil, nil
+	}
+
+	upgraded := len(oldService.Spec.IPFamilies) == 1 && len(service.Spec.IPFamilies) == 2
+	downgraded := len(oldService.Spec.IPFamilies) == 2 && len(service.Spec.IPFamilies) == 1
+
+	// CASE C:
+	if upgraded {
+		toAllocate := make(map[api.IPFamily]string)
+		// if secondary ip was named, just get it. if not add a marker
+		if len(service.Spec.ClusterIPs) < 2 {
+			service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* marker */)
+		}
+
+		toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1]
+
+		// allocate
+		allocated, err := al.allocIPs(service, toAllocate, dryRun)
+		// set if successful
+		if err == nil {
+			service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]]
+		}
+
+		return allocated, nil, err
+	}
+
+	// CASE D:
+	if downgraded {
+		toRelease = make(map[api.IPFamily]string)
+		toRelease[oldService.Spec.IPFamilies[1]] = oldService.Spec.ClusterIPs[1]
+		// note: we don't release clusterIP, this is left to clean up in the action itself
+		return nil, toRelease, err
+	}
+	// it was not an upgrade nor downgrade
+	return nil, nil, nil
+}
+
+func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, dryRun bool) (transaction, error) {
+	// The allocator tracks dry-run-ness internally.
+	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
+
+	txn := callbackTransaction{
+		commit: func() {
+			nodePortOp.Commit()
+			// We don't NEED to call Finish() here, but for that package says
+			// to, so for future-safety, we will.
+			nodePortOp.Finish()
+		},
+		revert: func() {
+			// Weirdly named but this will revert if commit wasn't called
+			nodePortOp.Finish()
+		},
+	}
+
+	// Update service from NodePort or LoadBalancer to ExternalName or ClusterIP, should release NodePort if exists.
+	if (oldService.Spec.Type == api.ServiceTypeNodePort || oldService.Spec.Type == api.ServiceTypeLoadBalancer) &&
+		(service.Spec.Type == api.ServiceTypeExternalName || service.Spec.Type == api.ServiceTypeClusterIP) {
+		releaseNodePorts(oldService, nodePortOp)
+	}
+
+	// Update service from any type to NodePort or LoadBalancer, should update NodePort.
+	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
+		if err := updateNodePorts(oldService, service, nodePortOp); err != nil {
+			txn.Revert()
+			return nil, err
+		}
+	}
+
+	// Handle ExternalTraffic related updates.
+	success, err := al.updateHealthCheckNodePort(oldService, service, nodePortOp)
+	if !success || err != nil {
+		txn.Revert()
+		return nil, err
+	}
+
+	return txn, nil
+}
+
+func releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) {
+	nodePorts := collectServiceNodePorts(service)
+
+	for _, nodePort := range nodePorts {
+		nodePortOp.ReleaseDeferred(nodePort)
+	}
+}
+
 func updateNodePorts(oldService, newService *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
 	oldNodePortsNumbers := collectServiceNodePorts(oldService)
 	newNodePorts := []ServiceNodePort{}
@@ -1095,12 +827,170 @@ func updateNodePorts(oldService, newService *api.Service, nodePortOp *portalloca
 	return nil
 }
 
-func releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) {
-	nodePorts := collectServiceNodePorts(service)
+// updateHealthCheckNodePort handles HealthCheckNodePort allocation/release
+// and adjusts HealthCheckNodePort during service update if needed.
+func (al *RESTAllocStuff) updateHealthCheckNodePort(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
+	neededHealthCheckNodePort := apiservice.NeedsHealthCheck(oldService)
+	oldHealthCheckNodePort := oldService.Spec.HealthCheckNodePort
 
-	for _, nodePort := range nodePorts {
-		nodePortOp.ReleaseDeferred(nodePort)
+	needsHealthCheckNodePort := apiservice.NeedsHealthCheck(service)
+
+	switch {
+	// Case 1: Transition from don't need HealthCheckNodePort to needs HealthCheckNodePort.
+	// Allocate a health check node port or attempt to reserve the user-specified one if provided.
+	// Insert health check node port into the service's HealthCheckNodePort field if needed.
+	case !neededHealthCheckNodePort && needsHealthCheckNodePort:
+		klog.Infof("Transition to LoadBalancer type service with ExternalTrafficPolicy=Local")
+		if err := al.allocHealthCheckNodePort(service, nodePortOp); err != nil {
+			return false, errors.NewInternalError(err)
+		}
+
+	// Case 2: Transition from needs HealthCheckNodePort to don't need HealthCheckNodePort.
+	// Free the existing healthCheckNodePort and clear the HealthCheckNodePort field.
+	case neededHealthCheckNodePort && !needsHealthCheckNodePort:
+		klog.Infof("Transition to non LoadBalancer type service or LoadBalancer type service with ExternalTrafficPolicy=Global")
+		klog.V(4).Infof("Releasing healthCheckNodePort: %d", oldHealthCheckNodePort)
+		nodePortOp.ReleaseDeferred(int(oldHealthCheckNodePort))
 	}
+	return true, nil
+}
+
+func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
+	al.releaseClusterIPs(svc)
+
+	for _, nodePort := range collectServiceNodePorts(svc) {
+		err := al.serviceNodePorts.Release(nodePort)
+		if err != nil {
+			// these should be caught by an eventual reconciliation / restart
+			utilruntime.HandleError(fmt.Errorf("Error releasing service %s node port %d: %v", svc.Name, nodePort, err))
+		}
+	}
+
+	if apiservice.NeedsHealthCheck(svc) {
+		nodePort := svc.Spec.HealthCheckNodePort
+		if nodePort > 0 {
+			err := al.serviceNodePorts.Release(int(nodePort))
+			if err != nil {
+				// these should be caught by an eventual reconciliation / restart
+				utilruntime.HandleError(fmt.Errorf("Error releasing service %s health check node port %d: %v", svc.Name, nodePort, err))
+			}
+		}
+	}
+}
+
+// releases allocated ClusterIPs for service that is about to be deleted
+func (al *RESTAllocStuff) releaseClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
+	// external name don't get ClusterIPs
+	if service.Spec.Type == api.ServiceTypeExternalName {
+		return nil, nil
+	}
+
+	// headless don't get ClusterIPs
+	if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone {
+		return nil, nil
+	}
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
+		return al.releaseClusterIP(service)
+	}
+
+	toRelease := make(map[api.IPFamily]string)
+	for _, ip := range service.Spec.ClusterIPs {
+		if netutils.IsIPv6String(ip) {
+			toRelease[api.IPv6Protocol] = ip
+		} else {
+			toRelease[api.IPv4Protocol] = ip
+		}
+	}
+	return al.releaseIPs(toRelease)
+}
+
+// for pre dual stack (gate == off). Hardwired to ClusterIP and ignores all new fields
+func (al *RESTAllocStuff) releaseClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
+	toRelease := make(map[api.IPFamily]string)
+
+	// we need to do that to handle cases where allocator is no longer configured on
+	// cluster
+	if netutils.IsIPv6String(service.Spec.ClusterIP) {
+		toRelease[api.IPv6Protocol] = service.Spec.ClusterIP
+	} else {
+		toRelease[api.IPv4Protocol] = service.Spec.ClusterIP
+	}
+
+	return al.releaseIPs(toRelease)
+}
+
+func isValidAddress(ctx context.Context, addr *api.EndpointAddress, pods rest.Getter) error {
+	if addr.TargetRef == nil {
+		return fmt.Errorf("Address has no target ref, skipping: %v", addr)
+	}
+	if genericapirequest.NamespaceValue(ctx) != addr.TargetRef.Namespace {
+		return fmt.Errorf("Address namespace doesn't match context namespace")
+	}
+	obj, err := pods.Get(ctx, addr.TargetRef.Name, &metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	pod, ok := obj.(*api.Pod)
+	if !ok {
+		return fmt.Errorf("failed to cast to pod: %v", obj)
+	}
+	if pod == nil {
+		return fmt.Errorf("pod is missing, skipping (%s/%s)", addr.TargetRef.Namespace, addr.TargetRef.Name)
+	}
+	for _, podIP := range pod.Status.PodIPs {
+		if podIP.IP == addr.IP {
+			return nil
+		}
+	}
+	return fmt.Errorf("pod ip(s) doesn't match endpoint ip, skipping: %v vs %s (%s/%s)", pod.Status.PodIPs, addr.IP, addr.TargetRef.Namespace, addr.TargetRef.Name)
+}
+
+// This is O(N), but we expect haystack to be small;
+// so small that we expect a linear search to be faster
+func containsNumber(haystack []int, needle int) bool {
+	for _, v := range haystack {
+		if v == needle {
+			return true
+		}
+	}
+	return false
+}
+
+// This is O(N), but we expect serviceNodePorts to be small;
+// so small that we expect a linear search to be faster
+func containsNodePort(serviceNodePorts []ServiceNodePort, serviceNodePort ServiceNodePort) bool {
+	for _, snp := range serviceNodePorts {
+		if snp == serviceNodePort {
+			return true
+		}
+	}
+	return false
+}
+
+// Loop through the service ports list, find one with the same port number and
+// NodePort specified, return this NodePort otherwise return 0.
+func findRequestedNodePort(port int, servicePorts []api.ServicePort) int {
+	for i := range servicePorts {
+		servicePort := servicePorts[i]
+		if port == int(servicePort.Port) && servicePort.NodePort != 0 {
+			return int(servicePort.NodePort)
+		}
+	}
+	return 0
+}
+
+func shouldAllocateNodePorts(service *api.Service) bool {
+	if service.Spec.Type == api.ServiceTypeNodePort {
+		return true
+	}
+	if service.Spec.Type == api.ServiceTypeLoadBalancer {
+		if utilfeature.DefaultFeatureGate.Enabled(features.ServiceLBNodePortControl) {
+			return *service.Spec.AllocateLoadBalancerNodePorts
+		}
+		return true
+	}
+	return false
 }
 
 func collectServiceNodePorts(service *api.Service) []int {
@@ -1113,3 +1003,112 @@ func collectServiceNodePorts(service *api.Service) []int {
 	}
 	return servicePorts
 }
+
+// tests if two preferred dual-stack service have matching ClusterIPFields
+// assumption: old service is a valid, default service (e.g., loaded from store)
+func isMatchingPreferDualStackClusterIPFields(oldService, service *api.Service) bool {
+	if oldService == nil {
+		return false
+	}
+
+	if service.Spec.IPFamilyPolicy == nil {
+		return false
+	}
+
+	// if type mutated then it is an update
+	// that needs to run through the entire process.
+	if oldService.Spec.Type != service.Spec.Type {
+		return false
+	}
+	// both must be type that gets an IP assigned
+	if service.Spec.Type != api.ServiceTypeClusterIP &&
+		service.Spec.Type != api.ServiceTypeNodePort &&
+		service.Spec.Type != api.ServiceTypeLoadBalancer {
+		return false
+	}
+
+	// both must be of IPFamilyPolicy==PreferDualStack
+	if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicyPreferDualStack {
+		return false
+	}
+
+	if oldService.Spec.IPFamilyPolicy != nil && *(oldService.Spec.IPFamilyPolicy) != api.IPFamilyPolicyPreferDualStack {
+		return false
+	}
+
+	if !sameClusterIPs(oldService, service) {
+		return false
+	}
+
+	if !sameIPFamilies(oldService, service) {
+		return false
+	}
+
+	// they match on
+	// Policy: preferDualStack
+	// ClusterIPs
+	// IPFamilies
+	return true
+}
+
+// Helper to avoid nil-checks all over.  Callers of this need to be checking
+// for an exact value.
+func getIPFamilyPolicy(svc *api.Service) api.IPFamilyPolicyType {
+	if svc.Spec.IPFamilyPolicy == nil {
+		return "" // callers need to handle this
+	}
+	return *svc.Spec.IPFamilyPolicy
+}
+
+func sameClusterIPs(lhs, rhs *api.Service) bool {
+	if len(rhs.Spec.ClusterIPs) != len(lhs.Spec.ClusterIPs) {
+		return false
+	}
+
+	for i, ip := range rhs.Spec.ClusterIPs {
+		if lhs.Spec.ClusterIPs[i] != ip {
+			return false
+		}
+	}
+
+	return true
+}
+
+func reducedClusterIPs(before, after *api.Service) bool {
+	if len(after.Spec.ClusterIPs) == 0 { // Not specified
+		return false
+	}
+	return len(after.Spec.ClusterIPs) < len(before.Spec.ClusterIPs)
+}
+
+func sameIPFamilies(lhs, rhs *api.Service) bool {
+	if len(rhs.Spec.IPFamilies) != len(lhs.Spec.IPFamilies) {
+		return false
+	}
+
+	for i, family := range rhs.Spec.IPFamilies {
+		if lhs.Spec.IPFamilies[i] != family {
+			return false
+		}
+	}
+
+	return true
+}
+
+func reducedIPFamilies(before, after *api.Service) bool {
+	if len(after.Spec.IPFamilies) == 0 { // Not specified
+		return false
+	}
+	return len(after.Spec.IPFamilies) < len(before.Spec.IPFamilies)
+}
+
+// Helper to get the IP family of a given IP.
+func familyOf(ip string) api.IPFamily {
+	if netutils.IsIPv4String(ip) {
+		return api.IPv4Protocol
+	}
+	if netutils.IsIPv6String(ip) {
+		return api.IPv6Protocol
+	}
+	return api.IPFamily("unknown")
+}

From d7c8557281d802a483b5b76f51348d4f5fec3342 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 23:20:09 -0700
Subject: [PATCH 73/79] Svc REST: Make update/releaseNodePorts methods

More consistent overall.
---
 pkg/registry/core/service/storage/alloc.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/registry/core/service/storage/alloc.go b/pkg/registry/core/service/storage/alloc.go
index 4969143b932..e2d1eb13234 100644
--- a/pkg/registry/core/service/storage/alloc.go
+++ b/pkg/registry/core/service/storage/alloc.go
@@ -746,12 +746,12 @@ func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, d
 	// Update service from NodePort or LoadBalancer to ExternalName or ClusterIP, should release NodePort if exists.
 	if (oldService.Spec.Type == api.ServiceTypeNodePort || oldService.Spec.Type == api.ServiceTypeLoadBalancer) &&
 		(service.Spec.Type == api.ServiceTypeExternalName || service.Spec.Type == api.ServiceTypeClusterIP) {
-		releaseNodePorts(oldService, nodePortOp)
+		al.releaseNodePorts(oldService, nodePortOp)
 	}
 
 	// Update service from any type to NodePort or LoadBalancer, should update NodePort.
 	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
-		if err := updateNodePorts(oldService, service, nodePortOp); err != nil {
+		if err := al.updateNodePorts(oldService, service, nodePortOp); err != nil {
 			txn.Revert()
 			return nil, err
 		}
@@ -767,7 +767,7 @@ func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, d
 	return txn, nil
 }
 
-func releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) {
+func (al *RESTAllocStuff) releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) {
 	nodePorts := collectServiceNodePorts(service)
 
 	for _, nodePort := range nodePorts {
@@ -775,7 +775,7 @@ func releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAlloca
 	}
 }
 
-func updateNodePorts(oldService, newService *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
+func (al *RESTAllocStuff) updateNodePorts(oldService, newService *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
 	oldNodePortsNumbers := collectServiceNodePorts(oldService)
 	newNodePorts := []ServiceNodePort{}
 	portAllocated := map[int]bool{}

From fe6f278ea137f45ff745510b60451096cc6a3782 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 23:21:20 -0700
Subject: [PATCH 74/79] Svc REST: Move isValidAddress to storage.go

---
 pkg/registry/core/service/storage/alloc.go   | 30 --------------------
 pkg/registry/core/service/storage/storage.go | 26 +++++++++++++++++
 2 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/pkg/registry/core/service/storage/alloc.go b/pkg/registry/core/service/storage/alloc.go
index e2d1eb13234..c2144f72023 100644
--- a/pkg/registry/core/service/storage/alloc.go
+++ b/pkg/registry/core/service/storage/alloc.go
@@ -17,15 +17,11 @@ limitations under the License.
 package storage
 
 import (
-	"context"
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
-	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
-	"k8s.io/apiserver/pkg/registry/rest"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 	apiservice "k8s.io/kubernetes/pkg/api/service"
@@ -920,32 +916,6 @@ func (al *RESTAllocStuff) releaseClusterIP(service *api.Service) (released map[a
 	return al.releaseIPs(toRelease)
 }
 
-func isValidAddress(ctx context.Context, addr *api.EndpointAddress, pods rest.Getter) error {
-	if addr.TargetRef == nil {
-		return fmt.Errorf("Address has no target ref, skipping: %v", addr)
-	}
-	if genericapirequest.NamespaceValue(ctx) != addr.TargetRef.Namespace {
-		return fmt.Errorf("Address namespace doesn't match context namespace")
-	}
-	obj, err := pods.Get(ctx, addr.TargetRef.Name, &metav1.GetOptions{})
-	if err != nil {
-		return err
-	}
-	pod, ok := obj.(*api.Pod)
-	if !ok {
-		return fmt.Errorf("failed to cast to pod: %v", obj)
-	}
-	if pod == nil {
-		return fmt.Errorf("pod is missing, skipping (%s/%s)", addr.TargetRef.Namespace, addr.TargetRef.Name)
-	}
-	for _, podIP := range pod.Status.PodIPs {
-		if podIP.IP == addr.IP {
-			return nil
-		}
-	}
-	return fmt.Errorf("pod ip(s) doesn't match endpoint ip, skipping: %v vs %s (%s/%s)", pod.Status.PodIPs, addr.IP, addr.TargetRef.Namespace, addr.TargetRef.Name)
-}
-
 // This is O(N), but we expect haystack to be small;
 // so small that we expect a linear search to be faster
 func containsNumber(haystack []int, needle int) bool {
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 056005dac7d..e457797279f 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -453,6 +453,32 @@ func (r *GenericREST) ResourceLocation(ctx context.Context, id string) (*url.URL
 	return nil, nil, errors.NewServiceUnavailable(fmt.Sprintf("no endpoints available for service %q", id))
 }
 
+func isValidAddress(ctx context.Context, addr *api.EndpointAddress, pods rest.Getter) error {
+	if addr.TargetRef == nil {
+		return fmt.Errorf("Address has no target ref, skipping: %v", addr)
+	}
+	if genericapirequest.NamespaceValue(ctx) != addr.TargetRef.Namespace {
+		return fmt.Errorf("Address namespace doesn't match context namespace")
+	}
+	obj, err := pods.Get(ctx, addr.TargetRef.Name, &metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	pod, ok := obj.(*api.Pod)
+	if !ok {
+		return fmt.Errorf("failed to cast to pod: %v", obj)
+	}
+	if pod == nil {
+		return fmt.Errorf("pod is missing, skipping (%s/%s)", addr.TargetRef.Namespace, addr.TargetRef.Name)
+	}
+	for _, podIP := range pod.Status.PodIPs {
+		if podIP.IP == addr.IP {
+			return nil
+		}
+	}
+	return fmt.Errorf("pod ip(s) doesn't match endpoint ip, skipping: %v vs %s (%s/%s)", pod.Status.PodIPs, addr.IP, addr.TargetRef.Namespace, addr.TargetRef.Name)
+}
+
 // normalizeClusterIPs adjust clusterIPs based on ClusterIP.  This must not
 // consider any other fields.
 func normalizeClusterIPs(oldSvc, newSvc *api.Service) {

From d5143bca84e0ee228d7429c057cdc79447d94f42 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 23:25:41 -0700
Subject: [PATCH 75/79] Svc REST: Rename GenericREST -> REST

This is consistent with every other registry.  Service is no longer the
oddball.
---
 pkg/registry/core/service/storage/storage.go  | 42 +++++++++----------
 .../core/service/storage/storage_test.go      |  4 +-
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index e457797279f..3ba0bc6d574 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -61,7 +61,7 @@ type PodStorage interface {
 	rest.Getter
 }
 
-type GenericREST struct {
+type REST struct {
 	*genericregistry.Store
 	primaryIPFamily   api.IPFamily
 	secondaryIPFamily api.IPFamily
@@ -72,14 +72,14 @@ type GenericREST struct {
 }
 
 var (
-	_ rest.CategoriesProvider     = &GenericREST{}
-	_ rest.ShortNamesProvider     = &GenericREST{}
-	_ rest.StorageVersionProvider = &GenericREST{}
-	_ rest.ResetFieldsStrategy    = &GenericREST{}
-	_ rest.Redirector             = &GenericREST{}
+	_ rest.CategoriesProvider     = &REST{}
+	_ rest.ShortNamesProvider     = &REST{}
+	_ rest.StorageVersionProvider = &REST{}
+	_ rest.ResetFieldsStrategy    = &REST{}
+	_ rest.Redirector             = &REST{}
 )
 
-// NewREST returns a RESTStorage object that will work against services.
+// NewREST returns a REST object that will work against services.
 func NewREST(
 	optsGetter generic.RESTOptionsGetter,
 	serviceIPFamily api.IPFamily,
@@ -87,7 +87,7 @@ func NewREST(
 	portAlloc portallocator.Interface,
 	endpoints EndpointsStorage,
 	pods PodStorage,
-	proxyTransport http.RoundTripper) (*GenericREST, *StatusREST, *svcreg.ProxyREST, error) {
+	proxyTransport http.RoundTripper) (*REST, *StatusREST, *svcreg.ProxyREST, error) {
 
 	strategy, _ := svcreg.StrategyForServiceCIDRs(ipAllocs[serviceIPFamily].CIDR(), len(ipAllocs) > 1)
 
@@ -119,7 +119,7 @@ func NewREST(
 	if len(ipAllocs) > 1 {
 		secondaryIPFamily = otherFamily(serviceIPFamily)
 	}
-	genericStore := &GenericREST{
+	genericStore := &REST{
 		Store:             store,
 		primaryIPFamily:   primaryIPFamily,
 		secondaryIPFamily: secondaryIPFamily,
@@ -146,21 +146,21 @@ func otherFamily(fam api.IPFamily) api.IPFamily {
 }
 
 var (
-	_ rest.ShortNamesProvider = &GenericREST{}
-	_ rest.CategoriesProvider = &GenericREST{}
+	_ rest.ShortNamesProvider = &REST{}
+	_ rest.CategoriesProvider = &REST{}
 )
 
 // ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource.
-func (r *GenericREST) ShortNames() []string {
+func (r *REST) ShortNames() []string {
 	return []string{"svc"}
 }
 
 // Categories implements the CategoriesProvider interface. Returns a list of categories a resource is part of.
-func (r *GenericREST) Categories() []string {
+func (r *REST) Categories() []string {
 	return []string{"all"}
 }
 
-// StatusREST implements the GenericREST endpoint for changing the status of a service.
+// StatusREST implements the REST endpoint for changing the status of a service.
 type StatusREST struct {
 	store *genericregistry.Store
 }
@@ -191,7 +191,7 @@ func (r *StatusREST) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
 // applies on Get, Create, and Update, but we need to distinguish between them.
 //
 // This will be called on both Service and ServiceList types.
-func (r *GenericREST) defaultOnRead(obj runtime.Object) {
+func (r *REST) defaultOnRead(obj runtime.Object) {
 	switch s := obj.(type) {
 	case *api.Service:
 		r.defaultOnReadService(s)
@@ -204,7 +204,7 @@ func (r *GenericREST) defaultOnRead(obj runtime.Object) {
 }
 
 // defaultOnReadServiceList defaults a ServiceList.
-func (r *GenericREST) defaultOnReadServiceList(serviceList *api.ServiceList) {
+func (r *REST) defaultOnReadServiceList(serviceList *api.ServiceList) {
 	if serviceList == nil {
 		return
 	}
@@ -215,7 +215,7 @@ func (r *GenericREST) defaultOnReadServiceList(serviceList *api.ServiceList) {
 }
 
 // defaultOnReadService defaults a single Service.
-func (r *GenericREST) defaultOnReadService(service *api.Service) {
+func (r *REST) defaultOnReadService(service *api.Service) {
 	if service == nil {
 		return
 	}
@@ -293,7 +293,7 @@ func (r *GenericREST) defaultOnReadService(service *api.Service) {
 	}
 }
 
-func (r *GenericREST) afterDelete(obj runtime.Object, options *metav1.DeleteOptions) {
+func (r *REST) afterDelete(obj runtime.Object, options *metav1.DeleteOptions) {
 	svc := obj.(*api.Service)
 
 	// Normally this defaulting is done automatically, but the hook (Decorator)
@@ -319,7 +319,7 @@ func (r *GenericREST) afterDelete(obj runtime.Object, options *metav1.DeleteOpti
 	}
 }
 
-func (r *GenericREST) beginCreate(ctx context.Context, obj runtime.Object, options *metav1.CreateOptions) (genericregistry.FinishFunc, error) {
+func (r *REST) beginCreate(ctx context.Context, obj runtime.Object, options *metav1.CreateOptions) (genericregistry.FinishFunc, error) {
 	svc := obj.(*api.Service)
 
 	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
@@ -349,7 +349,7 @@ func (r *GenericREST) beginCreate(ctx context.Context, obj runtime.Object, optio
 	return finish, nil
 }
 
-func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Object, options *metav1.UpdateOptions) (genericregistry.FinishFunc, error) {
+func (r *REST) beginUpdate(ctx context.Context, obj, oldObj runtime.Object, options *metav1.UpdateOptions) (genericregistry.FinishFunc, error) {
 	newSvc := obj.(*api.Service)
 	oldSvc := oldObj.(*api.Service)
 
@@ -381,7 +381,7 @@ func (r *GenericREST) beginUpdate(ctx context.Context, obj, oldObj runtime.Objec
 }
 
 // ResourceLocation returns a URL to which one can send traffic for the specified service.
-func (r *GenericREST) ResourceLocation(ctx context.Context, id string) (*url.URL, http.RoundTripper, error) {
+func (r *REST) ResourceLocation(ctx context.Context, id string) (*url.URL, http.RoundTripper, error) {
 	// Allow ID as "svcname", "svcname:port", or "scheme:svcname:port".
 	svcScheme, svcName, portStr, valid := utilnet.SplitSchemeNamePort(id)
 	if !valid {
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index a6000cc700f..716deac191d 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -147,7 +147,7 @@ func makePortAllocator(ports machineryutilnet.PortRange) portallocator.Interface
 // wrapperRESTForTests is a *trivial* wrapper for the real REST, which allows us to do
 // things that are specifically to enhance test safety.
 type wrapperRESTForTests struct {
-	*GenericREST
+	*REST
 }
 
 func (f *wrapperRESTForTests) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
@@ -155,7 +155,7 @@ func (f *wrapperRESTForTests) Create(ctx context.Context, obj runtime.Object, cr
 	// are not going to propagate to verification code, which used to happen
 	// resulting in tests that passed when they shouldn't have.
 	obj = obj.DeepCopyObject()
-	return f.GenericREST.Create(ctx, obj, createValidation, options)
+	return f.REST.Create(ctx, obj, createValidation, options)
 }
 
 //

From 5847426e5e20ffbeb091bdb331d32e4e9067e393 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Tue, 24 Aug 2021 23:47:59 -0700
Subject: [PATCH 76/79] Svc REST: Use types for safer arg ordering

In all the places we pass (old, new) or (new, old), use wrapper-types to
make sure that we don't flip the order by accident.
---
 pkg/registry/core/service/storage/alloc.go    | 67 ++++++++++++-------
 pkg/registry/core/service/storage/storage.go  | 40 ++++++++---
 .../core/service/storage/storage_test.go      |  4 +-
 3 files changed, 74 insertions(+), 37 deletions(-)

diff --git a/pkg/registry/core/service/storage/alloc.go b/pkg/registry/core/service/storage/alloc.go
index c2144f72023..e0c7bc81ed0 100644
--- a/pkg/registry/core/service/storage/alloc.go
+++ b/pkg/registry/core/service/storage/alloc.go
@@ -65,7 +65,7 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 
 	// Ensure IP family fields are correctly initialized.  We do it here, since
 	// we want this to be visible even when dryRun == true.
-	if err := al.initIPFamilyFields(nil, service); err != nil {
+	if err := al.initIPFamilyFields(After{service}, Before{nil}); err != nil {
 		return nil, err
 	}
 
@@ -93,7 +93,9 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 
 // attempts to default service ip families according to cluster configuration
 // while ensuring that provided families are configured on cluster.
-func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) error {
+func (al *RESTAllocStuff) initIPFamilyFields(after After, before Before) error {
+	oldService, service := before.Service, after.Service
+
 	// can not do anything here
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil
@@ -113,7 +115,7 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 	// when:
 	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
 	// - adding or removing a secondary clusterIP or ipFamily
-	if isMatchingPreferDualStackClusterIPFields(oldService, service) {
+	if isMatchingPreferDualStackClusterIPFields(after, before) {
 		return nil // nothing more to do.
 	}
 
@@ -172,11 +174,11 @@ func (al *RESTAllocStuff) initIPFamilyFields(oldService, service *api.Service) e
 		} else {
 			// If the policy is anything but single-stack AND they reduced these
 			// fields, it's an error.  They need to specify policy.
-			if reducedClusterIPs(oldService, service) {
+			if reducedClusterIPs(After{service}, Before{oldService}) {
 				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
 					"must be 'SingleStack' to release the secondary cluster IP"))
 			}
-			if reducedIPFamilies(oldService, service) {
+			if reducedIPFamilies(After{service}, Before{oldService}) {
 				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
 					"must be 'SingleStack' to release the secondary IP family"))
 			}
@@ -565,12 +567,12 @@ func (al *RESTAllocStuff) allocHealthCheckNodePort(service *api.Service, nodePor
 	return nil
 }
 
-func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRun bool) (transaction, error) {
+func (al *RESTAllocStuff) allocateUpdate(after After, before Before, dryRun bool) (transaction, error) {
 	result := metaTransaction{}
 
 	// Ensure IP family fields are correctly initialized.  We do it here, since
 	// we want this to be visible even when dryRun == true.
-	if err := al.initIPFamilyFields(oldService, service); err != nil {
+	if err := al.initIPFamilyFields(after, before); err != nil {
 		return nil, err
 	}
 
@@ -578,7 +580,7 @@ func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRu
 	//TODO(thockin): validation should not pass with empty clusterIP, but it
 	//does (and is tested!).  Fixing that all is a big PR and will have to
 	//happen later.
-	if txn, err := al.txnUpdateClusterIPs(service, oldService, dryRun); err != nil {
+	if txn, err := al.txnUpdateClusterIPs(after, before, dryRun); err != nil {
 		result.Revert()
 		return nil, err
 	} else {
@@ -586,7 +588,7 @@ func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRu
 	}
 
 	// Allocate ports
-	if txn, err := al.txnUpdateNodePorts(service, oldService, dryRun); err != nil {
+	if txn, err := al.txnUpdateNodePorts(after, before, dryRun); err != nil {
 		result.Revert()
 		return nil, err
 	} else {
@@ -596,8 +598,10 @@ func (al *RESTAllocStuff) allocateUpdate(service, oldService *api.Service, dryRu
 	return result, nil
 }
 
-func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *api.Service, dryRun bool) (transaction, error) {
-	allocated, released, err := al.updateClusterIPs(oldService, service, dryRun)
+func (al *RESTAllocStuff) txnUpdateClusterIPs(after After, before Before, dryRun bool) (transaction, error) {
+	service := after.Service
+
+	allocated, released, err := al.updateClusterIPs(after, before, dryRun)
 	if err != nil {
 		return nil, err
 	}
@@ -634,7 +638,8 @@ func (al *RESTAllocStuff) txnUpdateClusterIPs(service *api.Service, oldService *
 // this func does not perform actual release of clusterIPs. it returns
 // a map[family]ip for the caller to release when everything else has
 // executed successfully
-func (al *RESTAllocStuff) updateClusterIPs(oldService *api.Service, service *api.Service, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
+func (al *RESTAllocStuff) updateClusterIPs(after After, before Before, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
+	oldService, service := before.Service, after.Service
 
 	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
 	// PreferDualStack services following a cluster change to/from
@@ -644,7 +649,7 @@ func (al *RESTAllocStuff) updateClusterIPs(oldService *api.Service, service *api
 	// when:
 	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
 	// - adding or removing a secondary clusterIP or ipFamily
-	if isMatchingPreferDualStackClusterIPFields(oldService, service) {
+	if isMatchingPreferDualStackClusterIPFields(after, before) {
 		return allocated, toRelease, nil // nothing more to do.
 	}
 
@@ -722,7 +727,9 @@ func (al *RESTAllocStuff) updateClusterIPs(oldService *api.Service, service *api
 	return nil, nil, nil
 }
 
-func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, dryRun bool) (transaction, error) {
+func (al *RESTAllocStuff) txnUpdateNodePorts(after After, before Before, dryRun bool) (transaction, error) {
+	oldService, service := before.Service, after.Service
+
 	// The allocator tracks dry-run-ness internally.
 	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
 
@@ -747,14 +754,14 @@ func (al *RESTAllocStuff) txnUpdateNodePorts(service, oldService *api.Service, d
 
 	// Update service from any type to NodePort or LoadBalancer, should update NodePort.
 	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
-		if err := al.updateNodePorts(oldService, service, nodePortOp); err != nil {
+		if err := al.updateNodePorts(After{service}, Before{oldService}, nodePortOp); err != nil {
 			txn.Revert()
 			return nil, err
 		}
 	}
 
 	// Handle ExternalTraffic related updates.
-	success, err := al.updateHealthCheckNodePort(oldService, service, nodePortOp)
+	success, err := al.updateHealthCheckNodePort(After{service}, Before{oldService}, nodePortOp)
 	if !success || err != nil {
 		txn.Revert()
 		return nil, err
@@ -771,7 +778,9 @@ func (al *RESTAllocStuff) releaseNodePorts(service *api.Service, nodePortOp *por
 	}
 }
 
-func (al *RESTAllocStuff) updateNodePorts(oldService, newService *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
+func (al *RESTAllocStuff) updateNodePorts(after After, before Before, nodePortOp *portallocator.PortAllocationOperation) error {
+	oldService, newService := before.Service, after.Service
+
 	oldNodePortsNumbers := collectServiceNodePorts(oldService)
 	newNodePorts := []ServiceNodePort{}
 	portAllocated := map[int]bool{}
@@ -825,7 +834,9 @@ func (al *RESTAllocStuff) updateNodePorts(oldService, newService *api.Service, n
 
 // updateHealthCheckNodePort handles HealthCheckNodePort allocation/release
 // and adjusts HealthCheckNodePort during service update if needed.
-func (al *RESTAllocStuff) updateHealthCheckNodePort(oldService, service *api.Service, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
+func (al *RESTAllocStuff) updateHealthCheckNodePort(after After, before Before, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
+	oldService, service := before.Service, after.Service
+
 	neededHealthCheckNodePort := apiservice.NeedsHealthCheck(oldService)
 	oldHealthCheckNodePort := oldService.Spec.HealthCheckNodePort
 
@@ -976,7 +987,9 @@ func collectServiceNodePorts(service *api.Service) []int {
 
 // tests if two preferred dual-stack service have matching ClusterIPFields
 // assumption: old service is a valid, default service (e.g., loaded from store)
-func isMatchingPreferDualStackClusterIPFields(oldService, service *api.Service) bool {
+func isMatchingPreferDualStackClusterIPFields(after After, before Before) bool {
+	oldService, service := before.Service, after.Service
+
 	if oldService == nil {
 		return false
 	}
@@ -1044,11 +1057,13 @@ func sameClusterIPs(lhs, rhs *api.Service) bool {
 	return true
 }
 
-func reducedClusterIPs(before, after *api.Service) bool {
-	if len(after.Spec.ClusterIPs) == 0 { // Not specified
+func reducedClusterIPs(after After, before Before) bool {
+	oldSvc, newSvc := before.Service, after.Service
+
+	if len(newSvc.Spec.ClusterIPs) == 0 { // Not specified
 		return false
 	}
-	return len(after.Spec.ClusterIPs) < len(before.Spec.ClusterIPs)
+	return len(newSvc.Spec.ClusterIPs) < len(oldSvc.Spec.ClusterIPs)
 }
 
 func sameIPFamilies(lhs, rhs *api.Service) bool {
@@ -1065,11 +1080,13 @@ func sameIPFamilies(lhs, rhs *api.Service) bool {
 	return true
 }
 
-func reducedIPFamilies(before, after *api.Service) bool {
-	if len(after.Spec.IPFamilies) == 0 { // Not specified
+func reducedIPFamilies(after After, before Before) bool {
+	oldSvc, newSvc := before.Service, after.Service
+
+	if len(newSvc.Spec.IPFamilies) == 0 { // Not specified
 		return false
 	}
-	return len(after.Spec.IPFamilies) < len(before.Spec.IPFamilies)
+	return len(newSvc.Spec.IPFamilies) < len(oldSvc.Spec.IPFamilies)
 }
 
 // Helper to get the IP family of a given IP.
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 3ba0bc6d574..6fdc002ed91 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -186,6 +186,25 @@ func (r *StatusREST) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set {
 	return r.store.GetResetFields()
 }
 
+// We have a lot of functions that take a pair of "before" and "after" or
+// "oldSvc" and "newSvc" args.  Convention across the codebase is to pass them
+// as (new, old), but it's easy to screw up when they are the same type.
+//
+// These types force us to pay attention.  If the order of the arguments
+// matters, please receive them as:
+//    func something(after After, before Before) {
+//        oldSvc, newSvc := before.Service, after.Service
+//
+// If the order of arguments DOES NOT matter, please receive them as:
+//    func something(lhs, rhs *api.Service) {
+
+type Before struct {
+	*api.Service
+}
+type After struct {
+	*api.Service
+}
+
 // defaultOnRead sets interlinked fields that were not previously set on read.
 // We can't do this in the normal defaulting path because that same logic
 // applies on Get, Create, and Update, but we need to distinguish between them.
@@ -222,8 +241,7 @@ func (r *REST) defaultOnReadService(service *api.Service) {
 
 	// We might find Services that were written before ClusterIP became plural.
 	// We still want to present a consistent view of them.
-	// NOTE: the args are (old, new)
-	normalizeClusterIPs(nil, service)
+	normalizeClusterIPs(After{service}, Before{nil})
 
 	// The rest of this does not apply unless dual-stack is enabled.
 	if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
@@ -324,8 +342,7 @@ func (r *REST) beginCreate(ctx context.Context, obj runtime.Object, options *met
 
 	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
 	// early, before anyone looks at them.
-	// NOTE: the args are (old, new)
-	normalizeClusterIPs(nil, svc)
+	normalizeClusterIPs(After{svc}, Before{nil})
 
 	// Allocate IPs and ports. If we had a transactional store, this would just
 	// be part of the larger transaction.  We don't have that, so we have to do
@@ -355,15 +372,14 @@ func (r *REST) beginUpdate(ctx context.Context, obj, oldObj runtime.Object, opti
 
 	// Fix up allocated values that the client may have not specified (for
 	// idempotence).
-	patchAllocatedValues(newSvc, oldSvc)
+	patchAllocatedValues(After{newSvc}, Before{oldSvc})
 
 	// Make sure ClusterIP and ClusterIPs are in sync.  This has to happen
 	// early, before anyone looks at them.
-	// NOTE: the args are (old, new)
-	normalizeClusterIPs(oldSvc, newSvc)
+	normalizeClusterIPs(After{newSvc}, Before{oldSvc})
 
 	// Allocate and initialize fields.
-	txn, err := r.alloc.allocateUpdate(newSvc, oldSvc, dryrun.IsDryRun(options.DryRun))
+	txn, err := r.alloc.allocateUpdate(After{newSvc}, Before{oldSvc}, dryrun.IsDryRun(options.DryRun))
 	if err != nil {
 		return nil, err
 	}
@@ -481,7 +497,9 @@ func isValidAddress(ctx context.Context, addr *api.EndpointAddress, pods rest.Ge
 
 // normalizeClusterIPs adjust clusterIPs based on ClusterIP.  This must not
 // consider any other fields.
-func normalizeClusterIPs(oldSvc, newSvc *api.Service) {
+func normalizeClusterIPs(after After, before Before) {
+	oldSvc, newSvc := before.Service, after.Service
+
 	// In all cases here, we don't need to over-think the inputs.  Validation
 	// will be called on the new object soon enough.  All this needs to do is
 	// try to divine what user meant with these linked fields. The below
@@ -546,7 +564,9 @@ func normalizeClusterIPs(oldSvc, newSvc *api.Service) {
 // preserving values that we allocated on their behalf.  For example, they
 // might create a Service without specifying the ClusterIP, in which case we
 // allocate one.  If they resubmit that same YAML, we want it to succeed.
-func patchAllocatedValues(newSvc, oldSvc *api.Service) {
+func patchAllocatedValues(after After, before Before) {
+	oldSvc, newSvc := before.Service, after.Service
+
 	if needsClusterIP(oldSvc) && needsClusterIP(newSvc) {
 		if newSvc.Spec.ClusterIP == "" {
 			newSvc.Spec.ClusterIP = oldSvc.Spec.ClusterIP
diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 716deac191d..59bc4c03f15 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -402,7 +402,7 @@ func TestNormalizeClusterIPs(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			normalizeClusterIPs(tc.oldService, tc.newService)
+			normalizeClusterIPs(After{tc.newService}, Before{tc.oldService})
 
 			if tc.newService == nil {
 				t.Fatalf("unexpected new service to be nil")
@@ -506,7 +506,7 @@ func TestPatchAllocatedValues(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			update := tc.update.DeepCopy()
-			patchAllocatedValues(update, tc.before)
+			patchAllocatedValues(After{update}, Before{tc.before})
 
 			beforeIP := tc.before.Spec.ClusterIP
 			updateIP := update.Spec.ClusterIP

From 03d11c53a8af07619441b6fc401e8b9141544968 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 25 Aug 2021 13:57:51 -0700
Subject: [PATCH 77/79] Svc REST: Rename RESTAllocStuff

This was a dumb placeholder name.
---
 pkg/registry/core/service/storage/alloc.go   | 48 ++++++++++----------
 pkg/registry/core/service/storage/storage.go |  2 +-
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/pkg/registry/core/service/storage/alloc.go b/pkg/registry/core/service/storage/alloc.go
index e0c7bc81ed0..b80dbe25bfb 100644
--- a/pkg/registry/core/service/storage/alloc.go
+++ b/pkg/registry/core/service/storage/alloc.go
@@ -33,9 +33,9 @@ import (
 	netutils "k8s.io/utils/net"
 )
 
-// RESTAllocStuff is a temporary struct to facilitate the flattening of service
-// REST layers.  It will be cleaned up over a series of commits.
-type RESTAllocStuff struct {
+// Allocators encapsulates the various allocators (IPs, ports) used in
+// Services.
+type Allocators struct {
 	serviceIPAllocatorsByFamily map[api.IPFamily]ipallocator.Interface
 	defaultServiceIPFamily      api.IPFamily // --service-cluster-ip-range[0]
 	serviceNodePorts            portallocator.Interface
@@ -52,15 +52,15 @@ type ServiceNodePort struct {
 }
 
 // This is a trasitionary function to facilitate service REST flattening.
-func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) RESTAllocStuff {
-	return RESTAllocStuff{
+func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) Allocators {
+	return Allocators{
 		defaultServiceIPFamily:      defaultFamily,
 		serviceIPAllocatorsByFamily: ipAllocs,
 		serviceNodePorts:            portAlloc,
 	}
 }
 
-func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (transaction, error) {
+func (al *Allocators) allocateCreate(service *api.Service, dryRun bool) (transaction, error) {
 	result := metaTransaction{}
 
 	// Ensure IP family fields are correctly initialized.  We do it here, since
@@ -93,7 +93,7 @@ func (al *RESTAllocStuff) allocateCreate(service *api.Service, dryRun bool) (tra
 
 // attempts to default service ip families according to cluster configuration
 // while ensuring that provided families are configured on cluster.
-func (al *RESTAllocStuff) initIPFamilyFields(after After, before Before) error {
+func (al *Allocators) initIPFamilyFields(after After, before Before) error {
 	oldService, service := before.Service, after.Service
 
 	// can not do anything here
@@ -302,7 +302,7 @@ func (al *RESTAllocStuff) initIPFamilyFields(after After, before Before) error {
 	return nil
 }
 
-func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool) (transaction, error) {
+func (al *Allocators) txnAllocClusterIPs(service *api.Service, dryRun bool) (transaction, error) {
 	// clusterIPs that were allocated may need to be released in case of
 	// failure at a higher level.
 	toReleaseClusterIPs, err := al.allocClusterIPs(service, dryRun)
@@ -326,7 +326,7 @@ func (al *RESTAllocStuff) txnAllocClusterIPs(service *api.Service, dryRun bool)
 }
 
 // allocates ClusterIPs for a service
-func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
+func (al *Allocators) allocClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
 	// external name don't get ClusterIPs
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil, nil
@@ -387,7 +387,7 @@ func (al *RESTAllocStuff) allocClusterIPs(service *api.Service, dryRun bool) (ma
 
 // standard allocator for dualstackgate==Off, hard wired dependency
 // and ignores policy, families and clusterIPs
-func (al *RESTAllocStuff) allocClusterIP(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
+func (al *Allocators) allocClusterIP(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
 	toAlloc := make(map[api.IPFamily]string)
 
 	// get clusterIP.. empty string if user did not specify an ip
@@ -404,7 +404,7 @@ func (al *RESTAllocStuff) allocClusterIP(service *api.Service, dryRun bool) (map
 	return allocated, err
 }
 
-func (al *RESTAllocStuff) allocIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
+func (al *Allocators) allocIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
 	allocated := make(map[api.IPFamily]string)
 
 	for family, ip := range toAlloc {
@@ -431,7 +431,7 @@ func (al *RESTAllocStuff) allocIPs(service *api.Service, toAlloc map[api.IPFamil
 }
 
 // releases clusterIPs per family
-func (al *RESTAllocStuff) releaseIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
+func (al *Allocators) releaseIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
 	if toRelease == nil {
 		return nil, nil
 	}
@@ -455,7 +455,7 @@ func (al *RESTAllocStuff) releaseIPs(toRelease map[api.IPFamily]string) (map[api
 	return released, nil
 }
 
-func (al *RESTAllocStuff) txnAllocNodePorts(service *api.Service, dryRun bool) (transaction, error) {
+func (al *Allocators) txnAllocNodePorts(service *api.Service, dryRun bool) (transaction, error) {
 	// The allocator tracks dry-run-ness internally.
 	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)
 
@@ -545,7 +545,7 @@ func initNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocatio
 }
 
 // allocHealthCheckNodePort allocates health check node port to service.
-func (al *RESTAllocStuff) allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
+func (al *Allocators) allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
 	healthCheckNodePort := service.Spec.HealthCheckNodePort
 	if healthCheckNodePort != 0 {
 		// If the request has a health check nodePort in mind, attempt to reserve it.
@@ -567,7 +567,7 @@ func (al *RESTAllocStuff) allocHealthCheckNodePort(service *api.Service, nodePor
 	return nil
 }
 
-func (al *RESTAllocStuff) allocateUpdate(after After, before Before, dryRun bool) (transaction, error) {
+func (al *Allocators) allocateUpdate(after After, before Before, dryRun bool) (transaction, error) {
 	result := metaTransaction{}
 
 	// Ensure IP family fields are correctly initialized.  We do it here, since
@@ -598,7 +598,7 @@ func (al *RESTAllocStuff) allocateUpdate(after After, before Before, dryRun bool
 	return result, nil
 }
 
-func (al *RESTAllocStuff) txnUpdateClusterIPs(after After, before Before, dryRun bool) (transaction, error) {
+func (al *Allocators) txnUpdateClusterIPs(after After, before Before, dryRun bool) (transaction, error) {
 	service := after.Service
 
 	allocated, released, err := al.updateClusterIPs(after, before, dryRun)
@@ -638,7 +638,7 @@ func (al *RESTAllocStuff) txnUpdateClusterIPs(after After, before Before, dryRun
 // this func does not perform actual release of clusterIPs. it returns
 // a map[family]ip for the caller to release when everything else has
 // executed successfully
-func (al *RESTAllocStuff) updateClusterIPs(after After, before Before, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
+func (al *Allocators) updateClusterIPs(after After, before Before, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
 	oldService, service := before.Service, after.Service
 
 	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
@@ -727,7 +727,7 @@ func (al *RESTAllocStuff) updateClusterIPs(after After, before Before, dryRun bo
 	return nil, nil, nil
 }
 
-func (al *RESTAllocStuff) txnUpdateNodePorts(after After, before Before, dryRun bool) (transaction, error) {
+func (al *Allocators) txnUpdateNodePorts(after After, before Before, dryRun bool) (transaction, error) {
 	oldService, service := before.Service, after.Service
 
 	// The allocator tracks dry-run-ness internally.
@@ -770,7 +770,7 @@ func (al *RESTAllocStuff) txnUpdateNodePorts(after After, before Before, dryRun
 	return txn, nil
 }
 
-func (al *RESTAllocStuff) releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) {
+func (al *Allocators) releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) {
 	nodePorts := collectServiceNodePorts(service)
 
 	for _, nodePort := range nodePorts {
@@ -778,7 +778,7 @@ func (al *RESTAllocStuff) releaseNodePorts(service *api.Service, nodePortOp *por
 	}
 }
 
-func (al *RESTAllocStuff) updateNodePorts(after After, before Before, nodePortOp *portallocator.PortAllocationOperation) error {
+func (al *Allocators) updateNodePorts(after After, before Before, nodePortOp *portallocator.PortAllocationOperation) error {
 	oldService, newService := before.Service, after.Service
 
 	oldNodePortsNumbers := collectServiceNodePorts(oldService)
@@ -834,7 +834,7 @@ func (al *RESTAllocStuff) updateNodePorts(after After, before Before, nodePortOp
 
 // updateHealthCheckNodePort handles HealthCheckNodePort allocation/release
 // and adjusts HealthCheckNodePort during service update if needed.
-func (al *RESTAllocStuff) updateHealthCheckNodePort(after After, before Before, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
+func (al *Allocators) updateHealthCheckNodePort(after After, before Before, nodePortOp *portallocator.PortAllocationOperation) (bool, error) {
 	oldService, service := before.Service, after.Service
 
 	neededHealthCheckNodePort := apiservice.NeedsHealthCheck(oldService)
@@ -862,7 +862,7 @@ func (al *RESTAllocStuff) updateHealthCheckNodePort(after After, before Before,
 	return true, nil
 }
 
-func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
+func (al *Allocators) releaseAllocatedResources(svc *api.Service) {
 	al.releaseClusterIPs(svc)
 
 	for _, nodePort := range collectServiceNodePorts(svc) {
@@ -886,7 +886,7 @@ func (al *RESTAllocStuff) releaseAllocatedResources(svc *api.Service) {
 }
 
 // releases allocated ClusterIPs for service that is about to be deleted
-func (al *RESTAllocStuff) releaseClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
+func (al *Allocators) releaseClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) {
 	// external name don't get ClusterIPs
 	if service.Spec.Type == api.ServiceTypeExternalName {
 		return nil, nil
@@ -913,7 +913,7 @@ func (al *RESTAllocStuff) releaseClusterIPs(service *api.Service) (released map[
 }
 
 // for pre dual stack (gate == off). Hardwired to ClusterIP and ignores all new fields
-func (al *RESTAllocStuff) releaseClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
+func (al *Allocators) releaseClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) {
 	toRelease := make(map[api.IPFamily]string)
 
 	// we need to do that to handle cases where allocator is no longer configured on
diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go
index 6fdc002ed91..4250a334ab4 100644
--- a/pkg/registry/core/service/storage/storage.go
+++ b/pkg/registry/core/service/storage/storage.go
@@ -65,7 +65,7 @@ type REST struct {
 	*genericregistry.Store
 	primaryIPFamily   api.IPFamily
 	secondaryIPFamily api.IPFamily
-	alloc             RESTAllocStuff
+	alloc             Allocators
 	endpoints         EndpointsStorage
 	pods              PodStorage
 	proxyTransport    http.RoundTripper

From e594dd428159e60c6f16571321ccb58d6a1d956e Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Thu, 2 Sep 2021 10:06:53 -0700
Subject: [PATCH 78/79] Svc REST: Convert FIXME to TODO

---
 pkg/registry/core/service/storage/storage_test.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go
index 59bc4c03f15..74c61981aaa 100644
--- a/pkg/registry/core/service/storage/storage_test.go
+++ b/pkg/registry/core/service/storage/storage_test.go
@@ -11815,10 +11815,8 @@ func TestFeatureInternalTrafficPolicy(t *testing.T) {
 	helpTestCreateUpdateDelete(t, testCases)
 }
 
-// FIXME: externalIPs, lbip,
-// lbsourceranges, externalname, PublishNotReadyAddresses,
-// ipfamilypolicy and list,
-// AllocateLoadBalancerNodePorts, LoadBalancerClass, status
+// TODO(thockin): We need to look at feature-tests for:
+//   externalIPs, lbip, lbsourceranges, externalname, PublishNotReadyAddresses, AllocateLoadBalancerNodePorts, LoadBalancerClass, status
 
 // this is local because it's not fully fleshed out enough for general use.
 func makePod(name string, ips ...string) api.Pod {

From 009aa36c89e1aca05cd09ad228f801aacae607fc Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 10 Sep 2021 21:40:22 -0700
Subject: [PATCH 79/79] Svc REST: Make transaction-accumulating funcs safe

Identified in review, these funcs are now more reslient to future
changes.
---
 pkg/registry/core/service/storage/alloc.go | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/pkg/registry/core/service/storage/alloc.go b/pkg/registry/core/service/storage/alloc.go
index b80dbe25bfb..45cc8154410 100644
--- a/pkg/registry/core/service/storage/alloc.go
+++ b/pkg/registry/core/service/storage/alloc.go
@@ -62,6 +62,13 @@ func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator
 
 func (al *Allocators) allocateCreate(service *api.Service, dryRun bool) (transaction, error) {
 	result := metaTransaction{}
+	success := false
+
+	defer func() {
+		if !success {
+			result.Revert()
+		}
+	}()
 
 	// Ensure IP family fields are correctly initialized.  We do it here, since
 	// we want this to be visible even when dryRun == true.
@@ -74,7 +81,6 @@ func (al *Allocators) allocateCreate(service *api.Service, dryRun bool) (transac
 	//does (and is tested!).  Fixing that all is a big PR and will have to
 	//happen later.
 	if txn, err := al.txnAllocClusterIPs(service, dryRun); err != nil {
-		result.Revert()
 		return nil, err
 	} else {
 		result = append(result, txn)
@@ -82,12 +88,12 @@ func (al *Allocators) allocateCreate(service *api.Service, dryRun bool) (transac
 
 	// Allocate ports
 	if txn, err := al.txnAllocNodePorts(service, dryRun); err != nil {
-		result.Revert()
 		return nil, err
 	} else {
 		result = append(result, txn)
 	}
 
+	success = true
 	return result, nil
 }
 
@@ -569,6 +575,13 @@ func (al *Allocators) allocHealthCheckNodePort(service *api.Service, nodePortOp
 
 func (al *Allocators) allocateUpdate(after After, before Before, dryRun bool) (transaction, error) {
 	result := metaTransaction{}
+	success := false
+
+	defer func() {
+		if !success {
+			result.Revert()
+		}
+	}()
 
 	// Ensure IP family fields are correctly initialized.  We do it here, since
 	// we want this to be visible even when dryRun == true.
@@ -581,7 +594,6 @@ func (al *Allocators) allocateUpdate(after After, before Before, dryRun bool) (t
 	//does (and is tested!).  Fixing that all is a big PR and will have to
 	//happen later.
 	if txn, err := al.txnUpdateClusterIPs(after, before, dryRun); err != nil {
-		result.Revert()
 		return nil, err
 	} else {
 		result = append(result, txn)
@@ -589,12 +601,12 @@ func (al *Allocators) allocateUpdate(after After, before Before, dryRun bool) (t
 
 	// Allocate ports
 	if txn, err := al.txnUpdateNodePorts(after, before, dryRun); err != nil {
-		result.Revert()
 		return nil, err
 	} else {
 		result = append(result, txn)
 	}
 
+	success = true
 	return result, nil
 }