From 4f8fb1d3ca5ac8cbacf814a75fbe92f7093bc23e Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Wed, 28 Oct 2020 10:31:15 -0700 Subject: [PATCH] Wipe some fields on service "type" updates Service has had a problem since forever: - User creates a service type=LoadBalancer - We silently allocate them a NodePort - User changes type to ClusterIP - We fail the operation because they did not clear NodePort They never asked for or used the NodePort! Dual-stack introduced some dependent fields that get auto-wiped on updates. This carries it further. If you squint, you can see Service as a big, messy discriminated union, with type as the discriminator. Ignoring fields for non-selected union-modes seems right. This introduces the potential for an apply loop. Specifically, we will accept YAML that we did not previously accept. Apply could see the field in local YAML and not in the server and repeatedly try to patch it in. But since that YAML is currently an error, it seems like a very low risk. Almost nobody actually specifies their own NodePort values. To mitigate this somewhat, we only auto-wipe on updates. The same YAML would fail to create. This is a little inconsistent. We could auto-wipe on create, too, at the risk of more potential impact. To do this properly, we need to know the old and new values, which means we can not do it in defaulting or conversion. So we do it in strategy. This change also adds unit tests and updates e2e tests to rely on and verify this behavior. --- api/openapi-spec/swagger.json | 16 +- pkg/apis/core/v1/zz_generated.conversion.go | 2 +- pkg/registry/core/service/BUILD | 1 + pkg/registry/core/service/strategy.go | 147 ++++++- pkg/registry/core/service/strategy_test.go | 399 ++++++++++-------- .../src/k8s.io/api/core/v1/generated.pb.go | 178 ++++---- .../src/k8s.io/api/core/v1/generated.proto | 155 ++++--- staging/src/k8s.io/api/core/v1/types.go | 161 ++++--- .../core/v1/types_swagger_doc_generated.go | 16 +- .../api/core/v1/zz_generated.deepcopy.go | 10 +- .../api/testdata/HEAD/core.v1.Service.json | 8 +- .../api/testdata/HEAD/core.v1.Service.pb | Bin 419 -> 440 bytes .../api/testdata/HEAD/core.v1.Service.yaml | 4 +- test/e2e/network/service.go | 21 +- 14 files changed, 679 insertions(+), 439 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index c78c7543c77..4424dfd08ec 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -10227,7 +10227,7 @@ "type": "string" }, "nodePort": { - "description": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", + "description": "The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not specified, a port will be allocated if this Service requires one. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", "format": "int32", "type": "integer" }, @@ -10254,11 +10254,11 @@ "description": "ServiceSpec describes the attributes that a user creates on a service.", "properties": { "clusterIP": { - "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", + "description": "clusterIP is the IP address of the service and is usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be blank) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, "clusterIPs": { - "description": "ClusterIPs identifies all the ClusterIPs assigned to this service. ClusterIPs are assigned or reserved based on the values of service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are allowed in ClusterIPs. The IPFamily of each ClusterIP must match values provided in service.spec.ipFamilies. Clients using ClusterIPs must keep it in sync with ClusterIP (if provided) by having ClusterIP matching first element of ClusterIPs.", + "description": "ClusterIPs is a list of IP addresses assigned to this service, and are usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be empty) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. If this field is not specified, it will be initialized from the clusterIP field. If this field is specified, clients must ensure that clusterIPs[0] and clusterIP have the same value.\n\nUnless the \"IPv6DualStack\" feature gate is enabled, this field is limited to one value, which must be the same as the clusterIP field. If the feature gate is enabled, this field may hold a maximum of two entries (dual-stack IPs, in either order). These IPs must correspond to the values of the ipFamilies field. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "items": { "type": "string" }, @@ -10273,7 +10273,7 @@ "type": "array" }, "externalName": { - "description": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", + "description": "externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be", "type": "string" }, "externalTrafficPolicy": { @@ -10281,12 +10281,12 @@ "type": "string" }, "healthCheckNodePort": { - "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", + "description": "healthCheckNodePort specifies the healthcheck nodePort for the service. This only applies when type is set to LoadBalancer and externalTrafficPolicy is set to Local. If a value is specified, is in-range, and is not in use, it will be used. If not specified, a value will be automatically allocated. External systems (e.g. load-balancers) can use this port to determine if a given node holds endpoints for this service or not. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type).", "format": "int32", "type": "integer" }, "ipFamilies": { - "description": "IPFamilies identifies all the IPFamilies assigned for this Service. If a value was not provided for IPFamilies it will be defaulted based on the cluster configuration and the value of service.spec.ipFamilyPolicy. A maximum of two values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is conditionally mutable: it allows for adding or removing a secondary IPFamily, but it does not allow changing the primary IPFamily of the service.", + "description": "IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this service, and is gated by the \"IPv6DualStack\" feature gate. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply to \"headless\" services. This field will be wiped when updating a Service to type ExternalName.\n\nThis field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field.", "items": { "type": "string" }, @@ -10294,7 +10294,7 @@ "x-kubernetes-list-type": "atomic" }, "ipFamilyPolicy": { - "description": "IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this Service will be considered SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs respectively.", + "description": "IPFamilyPolicy represents the dual-stack-ness requested or required by this Service, and is gated by the \"IPv6DualStack\" feature gate. If there is no value provided, then this field will be set to SingleStack. Services can be \"SingleStack\" (a single IP family), \"PreferDualStack\" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or \"RequireDualStack\" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName.", "type": "string" }, "loadBalancerIP": { @@ -10349,7 +10349,7 @@ "type": "array" }, "type": { - "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", + "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object or EndpointSlice objects. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a virtual IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the same endpoints as the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the same endpoints as the clusterIP. \"ExternalName\" aliases this service to the specified externalName. Several other fields do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "type": "string" } }, diff --git a/pkg/apis/core/v1/zz_generated.conversion.go b/pkg/apis/core/v1/zz_generated.conversion.go index e7e8951a51c..1ee8400f915 100644 --- a/pkg/apis/core/v1/zz_generated.conversion.go +++ b/pkg/apis/core/v1/zz_generated.conversion.go @@ -7593,8 +7593,8 @@ func autoConvert_v1_ServiceSpec_To_core_ServiceSpec(in *v1.ServiceSpec, out *cor out.HealthCheckNodePort = in.HealthCheckNodePort out.PublishNotReadyAddresses = in.PublishNotReadyAddresses out.SessionAffinityConfig = (*core.SessionAffinityConfig)(unsafe.Pointer(in.SessionAffinityConfig)) - out.IPFamilies = *(*[]core.IPFamily)(unsafe.Pointer(&in.IPFamilies)) out.TopologyKeys = *(*[]string)(unsafe.Pointer(&in.TopologyKeys)) + out.IPFamilies = *(*[]core.IPFamily)(unsafe.Pointer(&in.IPFamilies)) out.IPFamilyPolicy = (*core.IPFamilyPolicyType)(unsafe.Pointer(in.IPFamilyPolicy)) return nil } diff --git a/pkg/registry/core/service/BUILD b/pkg/registry/core/service/BUILD index 7974b50fe4a..b4cab5ac82b 100644 --- a/pkg/registry/core/service/BUILD +++ b/pkg/registry/core/service/BUILD @@ -23,6 +23,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/proxy:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go index a053f3f6335..96e34c1c119 100644 --- a/pkg/registry/core/service/strategy.go +++ b/pkg/registry/core/service/strategy.go @@ -20,8 +20,10 @@ import ( "context" "fmt" "net" + "reflect" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/storage/names" @@ -107,9 +109,7 @@ func (strategy svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runti normalizeClusterIPs(oldService, newService) dropServiceDisabledFields(newService, oldService) - // if service was converted from ClusterIP => ExternalName - // then clear ClusterIPs, IPFamilyPolicy and IPFamilies - clearClusterIPRelatedFields(newService, oldService) + dropTypeDependentFields(newService, oldService) trimFieldsForDualStackDowngrade(newService, oldService) } @@ -298,23 +298,136 @@ func sameStringSlice(a []string, b []string) bool { return true } -// clearClusterIPRelatedFields ensures a backward compatible behavior when the user uses -// an older client to convert a service from ClusterIP to ExternalName. We do that by removing -// the newly introduced fields. -func clearClusterIPRelatedFields(newService, oldService *api.Service) { - if newService.Spec.Type == api.ServiceTypeExternalName && oldService.Spec.Type != api.ServiceTypeExternalName { - // IMPORTANT: this function is always called AFTER ClusterIPs normalization - // which clears ClusterIPs according to ClusterIP. The below checks for ClusterIP - clusterIPReset := len(newService.Spec.ClusterIP) == 0 && len(oldService.Spec.ClusterIP) > 0 +// This is an unusual case. Service has a number of inter-related fields and +// in order to avoid breaking clients we try really hard to infer what users +// mean when they change them. +// +// Services are effectively a discriminated union, where `type` is the +// discriminator. Some fields just don't make sense with some types, so we +// clear them. +// +// As a rule, we almost never change user input. This can get tricky when APIs +// evolve and new dependent fields are added. This specific case includes +// fields that are allocated from a pool and need to be released. Anyone who +// is contemplating copying this pattern should think REALLY hard about almost +// any other option. +func dropTypeDependentFields(newSvc *api.Service, oldSvc *api.Service) { + // For now we are only wiping on updates. This minimizes potential + // confusion since many of the cases we are handling here are pretty niche. + if oldSvc == nil { + return + } - if clusterIPReset { - // reset other fields - newService.Spec.ClusterIP = "" - newService.Spec.ClusterIPs = nil - newService.Spec.IPFamilies = nil - newService.Spec.IPFamilyPolicy = nil + // In all of these cases we only want to wipe a field if we a) know it no + // longer applies; b) might have initialized it automatically; c) know the + // user did not ALSO try to change it (in which case it should fail in + // validation). + + // If the user is switching to a type that does not need a value in + // clusterIP/clusterIPs (even "None" counts as a value), we might be able + // to wipe some fields. + if needsClusterIP(oldSvc) && !needsClusterIP(newSvc) { + if sameClusterIPs(oldSvc, newSvc) { + // These will be deallocated later. + newSvc.Spec.ClusterIP = "" + newSvc.Spec.ClusterIPs = nil + } + if sameIPFamilies(oldSvc, newSvc) { + newSvc.Spec.IPFamilies = nil + } + if sameIPFamilyPolicy(oldSvc, newSvc) { + newSvc.Spec.IPFamilyPolicy = nil } } + + // If the user is switching to a type that doesn't use NodePorts AND they + // did not change any NodePort values, we can wipe them. They will be + // deallocated later. + if needsNodePort(oldSvc) && !needsNodePort(newSvc) && sameNodePorts(oldSvc, newSvc) { + for i := range newSvc.Spec.Ports { + newSvc.Spec.Ports[i].NodePort = 0 + } + } + + // If the user is switching to a case that doesn't use HealthCheckNodePort AND they + // did not change the HealthCheckNodePort value, we can wipe it. It will + // be deallocated later. + if needsHCNodePort(oldSvc) && !needsHCNodePort(newSvc) && sameHCNodePort(oldSvc, newSvc) { + newSvc.Spec.HealthCheckNodePort = 0 + } + + // NOTE: there are other fields like `selector` which we could wipe. + // Historically we did not wipe them and they are not allocated from + // finite pools, so we are (currently) choosing to leave them alone. +} + +func needsClusterIP(svc *api.Service) bool { + if svc.Spec.Type == api.ServiceTypeExternalName { + return false + } + return true +} + +func sameClusterIPs(oldSvc, newSvc *api.Service) bool { + sameSingular := oldSvc.Spec.ClusterIP == newSvc.Spec.ClusterIP + samePlural := sameStringSlice(oldSvc.Spec.ClusterIPs, newSvc.Spec.ClusterIPs) + return sameSingular && samePlural +} + +func sameIPFamilies(oldSvc, newSvc *api.Service) bool { + return reflect.DeepEqual(oldSvc.Spec.IPFamilies, newSvc.Spec.IPFamilies) +} + +func getIPFamilyPolicy(svc *api.Service) string { + if svc.Spec.IPFamilyPolicy == nil { + return "" + } + return string(*svc.Spec.IPFamilyPolicy) +} + +func sameIPFamilyPolicy(oldSvc, newSvc *api.Service) bool { + return getIPFamilyPolicy(oldSvc) == getIPFamilyPolicy(newSvc) +} + +func needsNodePort(svc *api.Service) bool { + if svc.Spec.Type == api.ServiceTypeNodePort || svc.Spec.Type == api.ServiceTypeLoadBalancer { + return true + } + return false +} + +func sameNodePorts(oldSvc, newSvc *api.Service) bool { + // Helper to make a set of NodePort values. + allNodePorts := func(svc *api.Service) sets.Int { + out := sets.NewInt() + for i := range svc.Spec.Ports { + if svc.Spec.Ports[i].NodePort != 0 { + out.Insert(int(svc.Spec.Ports[i].NodePort)) + } + } + return out + } + + oldPorts := allNodePorts(oldSvc) + newPorts := allNodePorts(newSvc) + + // Users can add, remove, or modify ports, as long as they don't add any + // net-new NodePorts. + return oldPorts.IsSuperset(newPorts) +} + +func needsHCNodePort(svc *api.Service) bool { + if svc.Spec.Type != api.ServiceTypeLoadBalancer { + return false + } + if svc.Spec.ExternalTrafficPolicy != api.ServiceExternalTrafficPolicyTypeLocal { + return false + } + return true +} + +func sameHCNodePort(oldSvc, newSvc *api.Service) bool { + return oldSvc.Spec.HealthCheckNodePort == newSvc.Spec.HealthCheckNodePort } // this func allows user to downgrade a service by just changing diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go index 843b386e4d9..f4377199621 100644 --- a/pkg/registry/core/service/strategy_test.go +++ b/pkg/registry/core/service/strategy_test.go @@ -173,8 +173,10 @@ func TestCheckGeneratedNameError(t *testing.T) { } } -func makeValidService() api.Service { - return api.Service{ +func makeValidService() *api.Service { + preferDual := api.IPFamilyPolicyPreferDualStack + + return &api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "valid", Namespace: "default", @@ -186,11 +188,35 @@ func makeValidService() api.Service { Selector: map[string]string{"key": "val"}, SessionAffinity: "None", Type: api.ServiceTypeClusterIP, - Ports: []api.ServicePort{{Name: "p", Protocol: "TCP", Port: 8675, TargetPort: intstr.FromInt(8675)}}, + Ports: []api.ServicePort{ + makeValidServicePort("p", "TCP", 8675), + makeValidServicePort("q", "TCP", 309), + }, + ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4", "5:6:7::8"}, + IPFamilyPolicy: &preferDual, + IPFamilies: []api.IPFamily{"IPv4", "IPv6"}, }, } } +func makeValidServicePort(name string, proto api.Protocol, port int32) api.ServicePort { + return api.ServicePort{ + Name: name, + Protocol: proto, + Port: port, + TargetPort: intstr.FromInt(int(port)), + } +} + +func makeValidServiceCustom(tweaks ...func(svc *api.Service)) *api.Service { + svc := makeValidService() + for _, fn := range tweaks { + fn(svc) + } + return svc +} + // TODO: This should be done on types that are not part of our API func TestBeforeUpdate(t *testing.T) { testCases := []struct { @@ -248,9 +274,9 @@ func TestBeforeUpdate(t *testing.T) { oldSvc := makeValidService() newSvc := makeValidService() - tc.tweakSvc(&oldSvc, &newSvc) + tc.tweakSvc(oldSvc, newSvc) ctx := genericapirequest.NewDefaultContext() - err := rest.BeforeUpdate(strategy, ctx, runtime.Object(&oldSvc), runtime.Object(&newSvc)) + err := rest.BeforeUpdate(strategy, ctx, runtime.Object(oldSvc), runtime.Object(newSvc)) if tc.expectErr && err == nil { t.Errorf("unexpected non-error for %q", tc.name) } @@ -278,14 +304,14 @@ func TestServiceStatusStrategy(t *testing.T) { }, }, } - testStatusStrategy.PrepareForUpdate(ctx, &newService, &oldService) + testStatusStrategy.PrepareForUpdate(ctx, newService, oldService) if newService.Status.LoadBalancer.Ingress[0].IP != "127.0.0.2" { t.Errorf("Service status updates should allow change of status fields") } if newService.Spec.SessionAffinity != "None" { t.Errorf("PrepareForUpdate should have preserved old spec") } - errs := testStatusStrategy.ValidateUpdate(ctx, &newService, &oldService) + errs := testStatusStrategy.ValidateUpdate(ctx, newService, oldService) if len(errs) != 0 { t.Errorf("Unexpected error %v", errs) } @@ -379,8 +405,7 @@ func TestDropDisabledField(t *testing.T) { old := tc.oldSvc.DeepCopy() // to test against user using IPFamily not set on cluster - svcStrategy := svcStrategy{ipFamilies: []api.IPFamily{api.IPv4Protocol}} - svcStrategy.dropServiceDisabledFields(tc.svc, tc.oldSvc) + dropServiceDisabledFields(tc.svc, tc.oldSvc) // old node should never be changed if !reflect.DeepEqual(tc.oldSvc, old) { @@ -704,193 +729,193 @@ func TestNormalizeClusterIPs(t *testing.T) { } -func TestClearClusterIPRelatedFields(t *testing.T) { - // - // NOTE the data fed to this test assums that ClusterIPs normalization is - // already done check PrepareFor*(..) strategy - // - singleStack := api.IPFamilyPolicySingleStack - requireDualStack := api.IPFamilyPolicyRequireDualStack - testCases := []struct { - name string - oldService *api.Service - newService *api.Service - shouldClear bool - }{ - { - name: "should clear, single stack converting to external name", - shouldClear: true, - - oldService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - IPFamilyPolicy: &singleStack, - ClusterIP: "10.0.0.4", - ClusterIPs: []string{"10.0.0.4"}, - IPFamilies: []api.IPFamily{api.IPv4Protocol}, - }, - }, - newService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeExternalName, - IPFamilyPolicy: &singleStack, - ClusterIP: "", - ClusterIPs: nil, - IPFamilies: []api.IPFamily{api.IPv4Protocol}, - }, - }, - }, - - { - name: "should clear, dual stack converting to external name(normalization removed all ips)", - shouldClear: true, - - oldService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - IPFamilyPolicy: &requireDualStack, - ClusterIP: "2000::1", - ClusterIPs: []string{"2000::1", "10.0.0.4"}, - IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, - }, - }, - newService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeExternalName, - IPFamilyPolicy: &singleStack, - ClusterIP: "", - ClusterIPs: nil, - IPFamilies: []api.IPFamily{api.IPv4Protocol}, - }, - }, - }, - - { - name: "should NOT clear, single stack converting to external name ClusterIPs was not cleared", - shouldClear: false, - - oldService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - IPFamilyPolicy: &singleStack, - ClusterIP: "2000::1", - ClusterIPs: []string{"2000::1"}, - IPFamilies: []api.IPFamily{api.IPv6Protocol}, - }, - }, - newService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeExternalName, - IPFamilyPolicy: &singleStack, - ClusterIP: "2000::1", - ClusterIPs: []string{"2000::1"}, - IPFamilies: []api.IPFamily{api.IPv4Protocol}, - }, - }, - }, - - { - name: "should NOT clear, dualstack cleared primary and changed ClusterIPs", - shouldClear: true, - - oldService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - IPFamilyPolicy: &requireDualStack, - ClusterIP: "2000::1", - ClusterIPs: []string{"2000::1", "10.0.0.4"}, - IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, - }, - }, - newService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeExternalName, - IPFamilyPolicy: &singleStack, - ClusterIP: "", - ClusterIPs: []string{"2000::1", "10.0.0.5"}, - IPFamilies: []api.IPFamily{api.IPv4Protocol}, - }, - }, - }, - { - name: "should clear, dualstack user removed ClusterIPs", - shouldClear: true, - - oldService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - IPFamilyPolicy: &requireDualStack, - ClusterIP: "2000::1", - ClusterIPs: []string{"2000::1", "10.0.0.4"}, - IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, - }, - }, - - newService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeExternalName, - IPFamilyPolicy: &singleStack, - ClusterIP: "", - ClusterIPs: nil, - IPFamilies: []api.IPFamily{api.IPv4Protocol}, - }, - }, - }, - { - name: "should NOT clear, dualstack service changing selector", - shouldClear: false, - - oldService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - Selector: map[string]string{"foo": "bar"}, - IPFamilyPolicy: &requireDualStack, - ClusterIP: "2000::1", - ClusterIPs: []string{"2000::1", "10.0.0.4"}, - IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, - }, - }, - newService: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - Selector: map[string]string{"foo": "baz"}, - IPFamilyPolicy: &singleStack, - ClusterIP: "2000::1", - ClusterIPs: []string{"2000::1", "10.0.0.4"}, - IPFamilies: []api.IPFamily{api.IPv4Protocol}, - }, - }, - }, +func TestDropTypeDependentFields(t *testing.T) { + // Tweaks used below. + setTypeExternalName := func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeExternalName + } + setTypeNodePort := func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeNodePort + } + setTypeClusterIP := func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeClusterIP + } + setTypeLoadBalancer := func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + } + clearClusterIPs := func(svc *api.Service) { + svc.Spec.ClusterIP = "" + svc.Spec.ClusterIPs = nil + } + changeClusterIPs := func(svc *api.Service) { + svc.Spec.ClusterIP += "0" + svc.Spec.ClusterIPs[0] += "0" + } + setNodePorts := func(svc *api.Service) { + for i := range svc.Spec.Ports { + svc.Spec.Ports[i].NodePort = int32(30000 + i) + } + } + changeNodePorts := func(svc *api.Service) { + for i := range svc.Spec.Ports { + svc.Spec.Ports[i].NodePort += 100 + } + } + clearIPFamilies := func(svc *api.Service) { + svc.Spec.IPFamilies = nil + } + changeIPFamilies := func(svc *api.Service) { + svc.Spec.IPFamilies[0] = svc.Spec.IPFamilies[1] + } + clearIPFamilyPolicy := func(svc *api.Service) { + svc.Spec.IPFamilyPolicy = nil + } + changeIPFamilyPolicy := func(svc *api.Service) { + single := api.IPFamilyPolicySingleStack + svc.Spec.IPFamilyPolicy = &single + } + addPort := func(svc *api.Service) { + svc.Spec.Ports = append(svc.Spec.Ports, makeValidServicePort("new", "TCP", 0)) + } + delPort := func(svc *api.Service) { + svc.Spec.Ports = svc.Spec.Ports[0 : len(svc.Spec.Ports)-1] + } + changePort := func(svc *api.Service) { + svc.Spec.Ports[0].Port += 100 + svc.Spec.Ports[0].Protocol = "UDP" + } + setHCNodePort := func(svc *api.Service) { + svc.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal + svc.Spec.HealthCheckNodePort = int32(32000) + } + changeHCNodePort := func(svc *api.Service) { + svc.Spec.HealthCheckNodePort += 100 + } + patches := func(fns ...func(svc *api.Service)) func(svc *api.Service) { + return func(svc *api.Service) { + for _, fn := range fns { + fn(svc) + } + } } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - clearClusterIPRelatedFields(testCase.newService, testCase.oldService) + testCases := []struct { + name string + svc *api.Service + patch func(svc *api.Service) + expect *api.Service + }{ + { // clusterIP cases + name: "don't clear clusterIP et al", + svc: makeValidService(), + patch: nil, + expect: makeValidService(), + }, { + name: "clear clusterIP et al", + svc: makeValidService(), + patch: setTypeExternalName, + expect: makeValidServiceCustom(setTypeExternalName, clearClusterIPs, clearIPFamilies, clearIPFamilyPolicy), + }, { + name: "don't clear changed clusterIP", + svc: makeValidService(), + patch: patches(setTypeExternalName, changeClusterIPs), + expect: makeValidServiceCustom(setTypeExternalName, changeClusterIPs, clearIPFamilies, clearIPFamilyPolicy), + }, { + name: "don't clear changed ipFamilies", + svc: makeValidService(), + patch: patches(setTypeExternalName, changeIPFamilies), + expect: makeValidServiceCustom(setTypeExternalName, clearClusterIPs, changeIPFamilies, clearIPFamilyPolicy), + }, { + name: "don't clear changed ipFamilyPolicy", + svc: makeValidService(), + patch: patches(setTypeExternalName, changeIPFamilyPolicy), + expect: makeValidServiceCustom(setTypeExternalName, clearClusterIPs, clearIPFamilies, changeIPFamilyPolicy), + }, { // nodePort cases + name: "don't clear nodePorts for type=NodePort", + svc: makeValidServiceCustom(setTypeNodePort, setNodePorts), + patch: nil, + expect: makeValidServiceCustom(setTypeNodePort, setNodePorts), + }, { + name: "don't clear nodePorts for type=LoadBalancer", + svc: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + patch: nil, + expect: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + }, { + name: "clear nodePorts", + svc: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + patch: setTypeClusterIP, + expect: makeValidService(), + }, { + name: "don't clear changed nodePorts", + svc: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + patch: patches(setTypeClusterIP, changeNodePorts), + expect: makeValidServiceCustom(setNodePorts, changeNodePorts), + }, { + name: "clear nodePorts when adding a port", + svc: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + patch: patches(setTypeClusterIP, addPort), + expect: makeValidServiceCustom(addPort), + }, { + name: "don't clear nodePorts when adding a port with NodePort", + svc: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + patch: patches(setTypeClusterIP, addPort, setNodePorts), + expect: makeValidServiceCustom(addPort, setNodePorts), + }, { + name: "clear nodePorts when removing a port", + svc: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + patch: patches(setTypeClusterIP, delPort), + expect: makeValidServiceCustom(delPort), + }, { + name: "clear nodePorts when changing a port", + svc: makeValidServiceCustom(setTypeLoadBalancer, setNodePorts), + patch: patches(setTypeClusterIP, changePort), + expect: makeValidServiceCustom(changePort), + }, { // healthCheckNodePort cases + name: "don't clear healthCheckNodePort for type=LoadBalancer", + svc: makeValidServiceCustom(setTypeLoadBalancer, setHCNodePort), + patch: nil, + expect: makeValidServiceCustom(setTypeLoadBalancer, setHCNodePort), + }, { + name: "clear healthCheckNodePort", + svc: makeValidServiceCustom(setTypeLoadBalancer, setHCNodePort), + patch: setTypeClusterIP, + expect: makeValidService(), + }, { + name: "don't clear changed healthCheckNodePort", + svc: makeValidServiceCustom(setTypeLoadBalancer, setHCNodePort), + patch: patches(setTypeClusterIP, changeHCNodePort), + expect: makeValidServiceCustom(setHCNodePort, changeHCNodePort), + }} - if testCase.shouldClear && len(testCase.newService.Spec.ClusterIPs) != 0 { - t.Fatalf("expected clusterIPs to be cleared") + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := tc.svc.DeepCopy() + if tc.patch != nil { + tc.patch(result) } - - if testCase.shouldClear && len(testCase.newService.Spec.IPFamilies) != 0 { - t.Fatalf("expected ipfamilies to be cleared") + dropTypeDependentFields(result, tc.svc) + if result.Spec.ClusterIP != tc.expect.Spec.ClusterIP { + t.Errorf("expected clusterIP %q, got %q", tc.expect.Spec.ClusterIP, result.Spec.ClusterIP) } - - if testCase.shouldClear && testCase.newService.Spec.IPFamilyPolicy != nil { - t.Fatalf("expected ipfamilypolicy to be cleared") + if !reflect.DeepEqual(result.Spec.ClusterIPs, tc.expect.Spec.ClusterIPs) { + t.Errorf("expected clusterIPs %q, got %q", tc.expect.Spec.ClusterIP, result.Spec.ClusterIP) } - - if !testCase.shouldClear && len(testCase.newService.Spec.ClusterIPs) == 0 { - t.Fatalf("expected clusterIPs NOT to be cleared") + if !reflect.DeepEqual(result.Spec.IPFamilies, tc.expect.Spec.IPFamilies) { + t.Errorf("expected ipFamilies %q, got %q", tc.expect.Spec.IPFamilies, result.Spec.IPFamilies) } - - if !testCase.shouldClear && len(testCase.newService.Spec.IPFamilies) == 0 { - t.Fatalf("expected ipfamilies NOT to be cleared") + if !reflect.DeepEqual(result.Spec.IPFamilyPolicy, tc.expect.Spec.IPFamilyPolicy) { + t.Errorf("expected ipFamilyPolicy %q, got %q", getIPFamilyPolicy(tc.expect), getIPFamilyPolicy(result)) } - - if !testCase.shouldClear && testCase.newService.Spec.IPFamilyPolicy == nil { - t.Fatalf("expected ipfamilypolicy NOT to be cleared") + for i := range result.Spec.Ports { + resultPort := result.Spec.Ports[i].NodePort + expectPort := tc.expect.Spec.Ports[i].NodePort + if resultPort != expectPort { + t.Errorf("failed %q: expected Ports[%d].NodePort %d, got %d", tc.name, i, expectPort, resultPort) + } + } + if result.Spec.HealthCheckNodePort != tc.expect.Spec.HealthCheckNodePort { + t.Errorf("failed %q: expected healthCheckNodePort %d, got %d", tc.name, tc.expect.Spec.HealthCheckNodePort, result.Spec.HealthCheckNodePort) } - }) } } diff --git a/staging/src/k8s.io/api/core/v1/generated.pb.go b/staging/src/k8s.io/api/core/v1/generated.pb.go index ae81d521cef..d76257b88b8 100644 --- a/staging/src/k8s.io/api/core/v1/generated.pb.go +++ b/staging/src/k8s.io/api/core/v1/generated.pb.go @@ -6087,7 +6087,7 @@ func init() { } var fileDescriptor_83c10c24ec417dc9 = []byte{ - // 13928 bytes of a gzipped FileDescriptorProto + // 13927 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x1c, 0xd9, 0x79, 0x18, 0xaa, 0x9e, 0xc1, 0x6b, 0x3e, 0xbc, 0x0f, 0x48, 0x2e, 0x88, 0x25, 0x09, 0x6e, 0x53, 0xe2, 0x72, 0xb5, 0xbb, 0xa0, 0xf6, 0x25, 0xad, 0x77, 0xa5, 0xb5, 0x00, 0x0c, 0x40, 0xce, 0x92, @@ -6871,94 +6871,94 @@ var fileDescriptor_83c10c24ec417dc9 = []byte{ 0x37, 0x61, 0xb6, 0xdd, 0xd9, 0x6c, 0xba, 0xe1, 0xce, 0xba, 0x1f, 0x31, 0x53, 0x20, 0x95, 0x96, 0x56, 0x84, 0x4e, 0x50, 0x31, 0x27, 0xaa, 0x39, 0x78, 0x38, 0x97, 0x02, 0x7a, 0x1b, 0x4e, 0x27, 0x16, 0x83, 0x70, 0x1e, 0x9f, 0xc8, 0x8f, 0x71, 0x5f, 0xcb, 0xaa, 0x20, 0xe2, 0x30, 0x64, 0x81, - 0x70, 0x76, 0x13, 0xe8, 0x15, 0x00, 0xb7, 0xbd, 0xea, 0xb4, 0xdc, 0x26, 0x7d, 0x2e, 0xce, 0xb0, - 0x75, 0x42, 0x9f, 0x0e, 0x50, 0xa9, 0xca, 0x52, 0x7a, 0x3e, 0x8b, 0x7f, 0xfb, 0x58, 0xc3, 0xa6, - 0x0f, 0x17, 0x2d, 0x20, 0x69, 0x38, 0x3b, 0x15, 0xdb, 0x3b, 0x6b, 0x51, 0x4b, 0x43, 0x6c, 0x60, - 0xa1, 0x2a, 0x4c, 0x08, 0x1a, 0xfb, 0x62, 0x31, 0x4c, 0x2b, 0xef, 0xee, 0x09, 0xd9, 0x8e, 0x5a, - 0x01, 0xc8, 0x2c, 0x61, 0x73, 0x9e, 0xa8, 0xff, 0xce, 0x0c, 0xd3, 0xde, 0xa2, 0x95, 0x35, 0xde, - 0x15, 0x7d, 0x16, 0xc6, 0xf4, 0x9d, 0x20, 0xee, 0xe1, 0xcb, 0xd9, 0xac, 0x9d, 0xb6, 0x63, 0x38, - 0xe7, 0xab, 0x76, 0x85, 0x0e, 0xc3, 0x06, 0x45, 0x9b, 0x40, 0xf6, 0x1c, 0xa1, 0x9b, 0x30, 0x52, - 0x6f, 0xba, 0xc4, 0x8b, 0x2a, 0xd5, 0x6e, 0xd1, 0xb9, 0x96, 0x05, 0x8e, 0x98, 0x74, 0x11, 0xd8, - 0x9c, 0x97, 0x61, 0x45, 0xc1, 0xfe, 0xb5, 0x02, 0xcc, 0xf7, 0x88, 0x92, 0x9f, 0xd0, 0x9f, 0x58, - 0x7d, 0xe9, 0x4f, 0x16, 0x65, 0xa2, 0xe0, 0xf5, 0x84, 0x68, 0x26, 0x91, 0x04, 0x38, 0x16, 0xd0, - 0x24, 0xf1, 0xfb, 0xb6, 0x67, 0xd7, 0x55, 0x30, 0x03, 0x3d, 0x3d, 0x32, 0x0c, 0xd5, 0xeb, 0x60, - 0xff, 0xef, 0xb5, 0x5c, 0x35, 0x9a, 0xfd, 0x95, 0x02, 0x9c, 0x56, 0x43, 0xf8, 0xcd, 0x3b, 0x70, - 0xb7, 0xd3, 0x03, 0x77, 0x0c, 0x4a, 0x48, 0xfb, 0x16, 0x0c, 0xf1, 0x70, 0x63, 0x7d, 0xf0, 0x89, - 0x97, 0xcc, 0xc8, 0x9c, 0x8a, 0x35, 0x31, 0xa2, 0x73, 0x7e, 0xaf, 0x05, 0x93, 0x1b, 0xcb, 0xd5, - 0x9a, 0x5f, 0xdf, 0x25, 0xd1, 0x22, 0xe7, 0xeb, 0xb1, 0xe0, 0xf9, 0xac, 0x87, 0xe4, 0xe5, 0xb2, - 0xb8, 0xc4, 0x8b, 0x30, 0xb0, 0xe3, 0x87, 0x51, 0xd2, 0x42, 0xe1, 0xba, 0x1f, 0x46, 0x98, 0x41, - 0xec, 0xdf, 0xb3, 0x60, 0x90, 0xa5, 0xc6, 0x97, 0xd2, 0x6c, 0x2b, 0x47, 0x9a, 0xdd, 0xcf, 0x77, - 0xa1, 0x97, 0x60, 0x88, 0x6c, 0x6d, 0x91, 0x7a, 0x24, 0x66, 0x55, 0xba, 0x94, 0x0f, 0xad, 0xb0, - 0x52, 0xca, 0xb8, 0xb0, 0xc6, 0xf8, 0x5f, 0x2c, 0x90, 0xd1, 0x5d, 0x28, 0x45, 0x6e, 0x8b, 0x2c, - 0x36, 0x1a, 0x42, 0xc7, 0xfb, 0x10, 0x6e, 0xf1, 0x1b, 0x92, 0x00, 0x8e, 0x69, 0xd9, 0x5f, 0x2a, - 0x00, 0xc4, 0x71, 0x5a, 0x7a, 0x7d, 0xe2, 0x52, 0x4a, 0xfb, 0x77, 0x39, 0x43, 0xfb, 0x87, 0x62, - 0x82, 0x19, 0xaa, 0x3f, 0x35, 0x4c, 0xc5, 0xbe, 0x86, 0x69, 0xe0, 0x28, 0xc3, 0xb4, 0x0c, 0xd3, - 0x71, 0x9c, 0x19, 0x33, 0xcc, 0x16, 0x7b, 0xcb, 0x6d, 0x24, 0x81, 0x38, 0x8d, 0x6f, 0x13, 0xb8, - 0xa8, 0xc2, 0x6d, 0x88, 0xbb, 0x86, 0x99, 0x10, 0xeb, 0xda, 0xd4, 0x1e, 0xe3, 0x14, 0xab, 0x37, - 0x0b, 0xb9, 0xea, 0xcd, 0x9f, 0xb0, 0xe0, 0x54, 0xb2, 0x1d, 0xe6, 0xd3, 0xf9, 0x45, 0x0b, 0x4e, - 0x33, 0x25, 0x2f, 0x6b, 0x35, 0xad, 0x52, 0x7e, 0xb1, 0x6b, 0x08, 0x91, 0x9c, 0x1e, 0xc7, 0xb1, - 0x0b, 0xd6, 0xb2, 0x48, 0xe3, 0xec, 0x16, 0xed, 0x7f, 0x57, 0x80, 0xd9, 0xbc, 0xd8, 0x23, 0xcc, - 0xc3, 0xc0, 0xb9, 0x5f, 0xdb, 0x25, 0xf7, 0x84, 0x1d, 0x77, 0xec, 0x61, 0xc0, 0x8b, 0xb1, 0x84, - 0x27, 0x03, 0x9f, 0x17, 0xfa, 0x0b, 0x7c, 0x8e, 0x76, 0x60, 0xfa, 0xde, 0x0e, 0xf1, 0x6e, 0x7b, - 0xa1, 0x13, 0xb9, 0xe1, 0x96, 0xcb, 0x14, 0xa2, 0x7c, 0xdd, 0xbc, 0x22, 0xad, 0xad, 0xef, 0x26, - 0x11, 0x0e, 0x0f, 0xe6, 0xcf, 0x1b, 0x05, 0x71, 0x97, 0xf9, 0x41, 0x82, 0xd3, 0x44, 0xd3, 0x71, - 0xe3, 0x07, 0x1e, 0x61, 0xdc, 0x78, 0xfb, 0x8b, 0x16, 0x9c, 0xcd, 0x4d, 0x56, 0x89, 0xae, 0xc0, - 0x88, 0xd3, 0x76, 0xb9, 0x4c, 0x59, 0x1c, 0xa3, 0x4c, 0x76, 0x51, 0xad, 0x70, 0x89, 0xb2, 0x82, - 0xaa, 0x24, 0xda, 0x85, 0xdc, 0x24, 0xda, 0x3d, 0x73, 0x62, 0xdb, 0xdf, 0x63, 0x81, 0xf0, 0x8e, - 0xec, 0xe3, 0xec, 0xfe, 0x14, 0x8c, 0xed, 0xa5, 0x73, 0xcb, 0x5c, 0xcc, 0x77, 0x17, 0x15, 0x19, - 0x65, 0x14, 0xaf, 0x64, 0xe4, 0x91, 0x31, 0x68, 0xd9, 0x0d, 0x10, 0xd0, 0x32, 0x61, 0x12, 0xd3, - 0xde, 0xbd, 0x79, 0x1e, 0xa0, 0xc1, 0x70, 0xb5, 0x4c, 0xe4, 0xea, 0x66, 0x2e, 0x2b, 0x08, 0xd6, - 0xb0, 0xec, 0x7f, 0x5d, 0x80, 0x51, 0x99, 0xcb, 0xa4, 0xe3, 0xf5, 0x23, 0xd7, 0x38, 0x52, 0x72, - 0x43, 0x96, 0xba, 0x9f, 0x12, 0xae, 0xc6, 0xe2, 0xa0, 0x38, 0x75, 0xbf, 0x04, 0xe0, 0x18, 0x87, - 0xee, 0xa2, 0xb0, 0xb3, 0xc9, 0xd0, 0x13, 0xbe, 0x7c, 0x35, 0x5e, 0x8c, 0x25, 0x1c, 0x7d, 0x02, - 0xa6, 0x78, 0xbd, 0xc0, 0x6f, 0x3b, 0xdb, 0x5c, 0x58, 0x3f, 0xa8, 0x9c, 0xf0, 0xa7, 0xd6, 0x12, - 0xb0, 0xc3, 0x83, 0xf9, 0x53, 0xc9, 0x32, 0xa6, 0x85, 0x4a, 0x51, 0x61, 0x36, 0x39, 0xbc, 0x11, - 0xba, 0xfb, 0x53, 0xa6, 0x3c, 0x31, 0x08, 0xeb, 0x78, 0xf6, 0x67, 0x01, 0xa5, 0xb3, 0xba, 0xa0, - 0xd7, 0xb9, 0x21, 0xa6, 0x1b, 0x90, 0x46, 0x37, 0xad, 0x94, 0xee, 0x6a, 0x2e, 0xdd, 0x70, 0x78, - 0x2d, 0xac, 0xea, 0xdb, 0xff, 0x5f, 0x11, 0xa6, 0x92, 0x8e, 0xc7, 0xe8, 0x3a, 0x0c, 0x71, 0xd6, - 0x43, 0x90, 0xef, 0x62, 0xf4, 0xa0, 0xb9, 0x2b, 0xb3, 0x43, 0x58, 0x70, 0x2f, 0xa2, 0x3e, 0x7a, - 0x13, 0x46, 0x1b, 0xfe, 0x3d, 0xef, 0x9e, 0x13, 0x34, 0x16, 0xab, 0x15, 0xb1, 0x9c, 0x33, 0x5f, - 0x61, 0xe5, 0x18, 0x4d, 0x77, 0x81, 0x66, 0x0a, 0xbe, 0x18, 0x84, 0x75, 0x72, 0x68, 0x83, 0x05, - 0xa1, 0xde, 0x72, 0xb7, 0xd7, 0x9c, 0x76, 0x37, 0xab, 0xfc, 0x65, 0x89, 0xa4, 0x51, 0x1e, 0x17, - 0x91, 0xaa, 0x39, 0x00, 0xc7, 0x84, 0xd0, 0xe7, 0x61, 0x26, 0xcc, 0x91, 0x0d, 0xe7, 0x25, 0xf9, - 0xea, 0x26, 0x2e, 0x5d, 0x7a, 0x8c, 0xbe, 0x8f, 0xb3, 0xa4, 0xc8, 0x59, 0xcd, 0xd8, 0x3f, 0x72, - 0x0a, 0x8c, 0x4d, 0x6c, 0xe4, 0x7c, 0xb4, 0x8e, 0x29, 0xe7, 0x23, 0x86, 0x11, 0xd2, 0x6a, 0x47, - 0xfb, 0x65, 0x37, 0xe8, 0x96, 0x09, 0x79, 0x45, 0xe0, 0xa4, 0x69, 0x4a, 0x08, 0x56, 0x74, 0xb2, - 0x13, 0x73, 0x16, 0xbf, 0x8e, 0x89, 0x39, 0x07, 0x4e, 0x30, 0x31, 0xe7, 0x3a, 0x0c, 0x6f, 0xbb, - 0x11, 0x26, 0x6d, 0x5f, 0x30, 0xfd, 0x99, 0xeb, 0xf0, 0x1a, 0x47, 0x49, 0xa7, 0x80, 0x13, 0x00, - 0x2c, 0x89, 0xa0, 0xd7, 0xd5, 0x0e, 0x1c, 0xca, 0x7f, 0x33, 0xa7, 0xb5, 0xf3, 0x99, 0x7b, 0x50, - 0xa4, 0xdf, 0x1c, 0x7e, 0xd8, 0xf4, 0x9b, 0xab, 0x32, 0x69, 0xe6, 0x48, 0xbe, 0x0b, 0x0d, 0xcb, - 0x89, 0xd9, 0x23, 0x55, 0xe6, 0x1d, 0x3d, 0xd1, 0x68, 0x29, 0xff, 0x24, 0x50, 0x39, 0x44, 0xfb, - 0x4c, 0x2f, 0xfa, 0x3d, 0x16, 0x9c, 0x6e, 0x67, 0xe5, 0xdc, 0x15, 0x8a, 0xec, 0x97, 0xfa, 0x4e, - 0xeb, 0x6b, 0x34, 0xc8, 0x04, 0x40, 0x99, 0x68, 0x38, 0xbb, 0x39, 0x3a, 0xd0, 0xc1, 0x66, 0x43, - 0x28, 0x54, 0x2f, 0xe5, 0xe4, 0x29, 0xed, 0x92, 0x9d, 0x74, 0x23, 0x23, 0x27, 0xe6, 0xfb, 0xf3, - 0x72, 0x62, 0xf6, 0x9d, 0x09, 0xf3, 0x75, 0x95, 0xa1, 0x74, 0x3c, 0x7f, 0x29, 0xf1, 0xfc, 0xa3, - 0x3d, 0xf3, 0x92, 0xbe, 0xae, 0xf2, 0x92, 0x76, 0x89, 0x30, 0xca, 0xb3, 0x8e, 0xf6, 0xcc, 0x46, - 0xaa, 0x65, 0x14, 0x9d, 0x3c, 0x9e, 0x8c, 0xa2, 0xc6, 0x55, 0xc3, 0x93, 0x5a, 0x3e, 0xdd, 0xe3, - 0xaa, 0x31, 0xe8, 0x76, 0xbf, 0x6c, 0x78, 0xf6, 0xd4, 0xe9, 0x87, 0xca, 0x9e, 0x7a, 0x47, 0xcf, - 0x46, 0x8a, 0x7a, 0xa4, 0xdb, 0xa4, 0x48, 0x7d, 0xe6, 0x20, 0xbd, 0xa3, 0x5f, 0x80, 0x33, 0xf9, - 0x74, 0xd5, 0x3d, 0x97, 0xa6, 0x9b, 0x79, 0x05, 0xa6, 0x72, 0x9b, 0x9e, 0x3a, 0x99, 0xdc, 0xa6, - 0xa7, 0x8f, 0x3d, 0xb7, 0xe9, 0x99, 0x13, 0xc8, 0x6d, 0xfa, 0xd8, 0x09, 0xe6, 0x36, 0xbd, 0xc3, - 0xac, 0x3f, 0x78, 0x8c, 0x19, 0x11, 0x11, 0x35, 0x3b, 0xfa, 0x66, 0x56, 0x20, 0x1a, 0xfe, 0x71, - 0x0a, 0x84, 0x63, 0x52, 0x19, 0x39, 0x53, 0x67, 0x1f, 0x41, 0xce, 0xd4, 0xf5, 0x38, 0x67, 0xea, - 0xd9, 0xfc, 0xa9, 0xce, 0xf0, 0x17, 0xc8, 0xc9, 0x94, 0x7a, 0x47, 0xcf, 0x70, 0xfa, 0x78, 0x17, - 0x11, 0x7f, 0x96, 0xe0, 0xb1, 0x4b, 0x5e, 0xd3, 0xd7, 0x78, 0x5e, 0xd3, 0x73, 0xf9, 0x27, 0x79, - 0xf2, 0xba, 0x33, 0xb2, 0x99, 0xd2, 0x7e, 0xa9, 0xd8, 0x7b, 0x2c, 0xf6, 0x6b, 0x4e, 0xbf, 0x54, - 0xf0, 0xbe, 0x74, 0xbf, 0x14, 0x08, 0xc7, 0xa4, 0xec, 0xef, 0x2b, 0xc0, 0x85, 0xee, 0xfb, 0x2d, - 0x96, 0xa6, 0x56, 0x63, 0x8d, 0x67, 0x42, 0x9a, 0xca, 0xdf, 0x6c, 0x31, 0x56, 0xdf, 0x61, 0xcd, - 0xae, 0xc1, 0xb4, 0x72, 0x34, 0x68, 0xba, 0xf5, 0xfd, 0xf5, 0xf8, 0xe5, 0xab, 0x9c, 0xb3, 0x6b, - 0x49, 0x04, 0x9c, 0xae, 0x83, 0x16, 0x61, 0xd2, 0x28, 0xac, 0x94, 0xc5, 0xdb, 0x4c, 0x89, 0x6f, - 0x6b, 0x26, 0x18, 0x27, 0xf1, 0xed, 0x2f, 0x5b, 0xf0, 0x58, 0x4e, 0x3a, 0xb2, 0xbe, 0xa3, 0x76, - 0x6d, 0xc1, 0x64, 0xdb, 0xac, 0xda, 0x23, 0xb8, 0x9f, 0x91, 0xf4, 0x4c, 0xf5, 0x35, 0x01, 0xc0, - 0x49, 0xa2, 0xf6, 0x9f, 0x5b, 0x70, 0xbe, 0xab, 0xe5, 0x1c, 0xc2, 0x70, 0x66, 0xbb, 0x15, 0x3a, - 0xcb, 0x01, 0x69, 0x10, 0x2f, 0x72, 0x9d, 0x66, 0xad, 0x4d, 0xea, 0x9a, 0x3c, 0x9c, 0x99, 0xa0, - 0x5d, 0x5b, 0xab, 0x2d, 0xa6, 0x31, 0x70, 0x4e, 0x4d, 0xb4, 0x0a, 0x28, 0x0d, 0x11, 0x33, 0xcc, - 0xa2, 0x08, 0xa7, 0xe9, 0xe1, 0x8c, 0x1a, 0xe8, 0x23, 0x30, 0xae, 0x2c, 0xf2, 0xb4, 0x19, 0x67, - 0x07, 0x3b, 0xd6, 0x01, 0xd8, 0xc4, 0x5b, 0xba, 0xf2, 0x9b, 0x7f, 0x70, 0xe1, 0x7d, 0xbf, 0xfd, - 0x07, 0x17, 0xde, 0xf7, 0xbb, 0x7f, 0x70, 0xe1, 0x7d, 0xdf, 0xf1, 0xe0, 0x82, 0xf5, 0x9b, 0x0f, - 0x2e, 0x58, 0xbf, 0xfd, 0xe0, 0x82, 0xf5, 0xbb, 0x0f, 0x2e, 0x58, 0xbf, 0xff, 0xe0, 0x82, 0xf5, - 0xa5, 0x3f, 0xbc, 0xf0, 0xbe, 0x4f, 0x15, 0xf6, 0x9e, 0xfb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, - 0x0f, 0xd0, 0xef, 0xf4, 0xac, 0x01, 0x01, 0x00, + 0x70, 0x76, 0x13, 0xf4, 0xf1, 0xa1, 0x05, 0x15, 0x0d, 0x67, 0xa7, 0x62, 0x9b, 0x65, 0x2d, 0xf2, + 0x68, 0x88, 0x0d, 0x2c, 0xf4, 0x0a, 0x80, 0xdb, 0x5e, 0x75, 0x5a, 0x6e, 0x93, 0x3e, 0x32, 0x67, + 0x58, 0x1d, 0xfa, 0xe0, 0x80, 0x4a, 0x55, 0x96, 0xd2, 0x53, 0x5d, 0xfc, 0xdb, 0xc7, 0x1a, 0x36, + 0xaa, 0xc2, 0x84, 0xf8, 0xb7, 0x2f, 0x16, 0xc3, 0xb4, 0xf2, 0xee, 0x9e, 0x90, 0x35, 0xd4, 0x0a, + 0x40, 0x66, 0x09, 0x9b, 0xf3, 0x44, 0xfd, 0x77, 0x66, 0x98, 0xf6, 0x16, 0xad, 0xac, 0xf1, 0xae, + 0xe8, 0xb3, 0x30, 0xa6, 0xef, 0x04, 0x71, 0x0f, 0x5f, 0xce, 0x66, 0xed, 0xb4, 0x1d, 0xc3, 0x39, + 0x5f, 0xb5, 0x2b, 0x74, 0x18, 0x36, 0x28, 0xda, 0x04, 0xb2, 0xe7, 0x08, 0xdd, 0x84, 0x91, 0x7a, + 0xd3, 0x25, 0x5e, 0x54, 0xa9, 0x76, 0x8b, 0xce, 0xb5, 0x2c, 0x70, 0xc4, 0xa4, 0x8b, 0xc0, 0xe6, + 0xbc, 0x0c, 0x2b, 0x0a, 0xf6, 0xaf, 0x15, 0x60, 0xbe, 0x47, 0x94, 0xfc, 0x84, 0xfe, 0xc4, 0xea, + 0x4b, 0x7f, 0xb2, 0x28, 0x13, 0x05, 0xaf, 0x27, 0x44, 0x33, 0x89, 0x24, 0xc0, 0xb1, 0x80, 0x26, + 0x89, 0xdf, 0xb7, 0x3d, 0xbb, 0xae, 0x82, 0x19, 0xe8, 0xe9, 0x91, 0x61, 0xa8, 0x5e, 0x07, 0xfb, + 0x7f, 0xaf, 0xe5, 0xaa, 0xd1, 0xec, 0xaf, 0x14, 0xe0, 0xb4, 0x1a, 0xc2, 0x6f, 0xde, 0x81, 0xbb, + 0x9d, 0x1e, 0xb8, 0x63, 0x50, 0x42, 0xda, 0xb7, 0x60, 0x88, 0x87, 0x1b, 0xeb, 0x83, 0x4f, 0xbc, + 0x64, 0x46, 0xe6, 0x54, 0xac, 0x89, 0x11, 0x9d, 0xf3, 0x7b, 0x2d, 0x98, 0xdc, 0x58, 0xae, 0xd6, + 0xfc, 0xfa, 0x2e, 0x89, 0x16, 0x39, 0x5f, 0x8f, 0x05, 0xcf, 0x67, 0x3d, 0x24, 0x2f, 0x97, 0xc5, + 0x25, 0x5e, 0x84, 0x81, 0x1d, 0x3f, 0x8c, 0x92, 0x16, 0x0a, 0xd7, 0xfd, 0x30, 0xc2, 0x0c, 0x62, + 0xff, 0x9e, 0x05, 0x83, 0x2c, 0x35, 0xbe, 0x94, 0x66, 0x5b, 0x39, 0xd2, 0xec, 0x7e, 0xbe, 0x0b, + 0xbd, 0x04, 0x43, 0x64, 0x6b, 0x8b, 0xd4, 0x23, 0x31, 0xab, 0xd2, 0xa5, 0x7c, 0x68, 0x85, 0x95, + 0x52, 0xc6, 0x85, 0x35, 0xc6, 0xff, 0x62, 0x81, 0x8c, 0xee, 0x42, 0x29, 0x72, 0x5b, 0x64, 0xb1, + 0xd1, 0x10, 0x3a, 0xde, 0x87, 0x70, 0x8b, 0xdf, 0x90, 0x04, 0x70, 0x4c, 0xcb, 0xfe, 0x52, 0x01, + 0x20, 0x8e, 0xd3, 0xd2, 0xeb, 0x13, 0x97, 0x52, 0xda, 0xbf, 0xcb, 0x19, 0xda, 0x3f, 0x14, 0x13, + 0xcc, 0x50, 0xfd, 0xa9, 0x61, 0x2a, 0xf6, 0x35, 0x4c, 0x03, 0x47, 0x19, 0xa6, 0x65, 0x98, 0x8e, + 0xe3, 0xcc, 0x98, 0x61, 0xb6, 0xd8, 0x5b, 0x6e, 0x23, 0x09, 0xc4, 0x69, 0x7c, 0x9b, 0xc0, 0x45, + 0x15, 0x6e, 0x43, 0xdc, 0x35, 0xcc, 0x84, 0x58, 0xd7, 0xa6, 0xf6, 0x18, 0xa7, 0x58, 0xbd, 0x59, + 0xc8, 0x55, 0x6f, 0xfe, 0x84, 0x05, 0xa7, 0x92, 0xed, 0x30, 0x9f, 0xce, 0x2f, 0x5a, 0x70, 0x9a, + 0x29, 0x79, 0x59, 0xab, 0x69, 0x95, 0xf2, 0x8b, 0x5d, 0x43, 0x88, 0xe4, 0xf4, 0x38, 0x8e, 0x5d, + 0xb0, 0x96, 0x45, 0x1a, 0x67, 0xb7, 0x68, 0xff, 0xbb, 0x02, 0xcc, 0xe6, 0xc5, 0x1e, 0x61, 0x1e, + 0x06, 0xce, 0xfd, 0xda, 0x2e, 0xb9, 0x27, 0xec, 0xb8, 0x63, 0x0f, 0x03, 0x5e, 0x8c, 0x25, 0x3c, + 0x19, 0xf8, 0xbc, 0xd0, 0x5f, 0xe0, 0x73, 0xb4, 0x03, 0xd3, 0xf7, 0x76, 0x88, 0x77, 0xdb, 0x0b, + 0x9d, 0xc8, 0x0d, 0xb7, 0x5c, 0xa6, 0x10, 0xe5, 0xeb, 0xe6, 0x15, 0x69, 0x6d, 0x7d, 0x37, 0x89, + 0x70, 0x78, 0x30, 0x7f, 0xde, 0x28, 0x88, 0xbb, 0xcc, 0x0f, 0x12, 0x9c, 0x26, 0x9a, 0x8e, 0x1b, + 0x3f, 0xf0, 0x08, 0xe3, 0xc6, 0xdb, 0x5f, 0xb4, 0xe0, 0x6c, 0x6e, 0xb2, 0x4a, 0x74, 0x05, 0x46, + 0x9c, 0xb6, 0xcb, 0x65, 0xca, 0xe2, 0x18, 0x65, 0xb2, 0x8b, 0x6a, 0x85, 0x4b, 0x94, 0x15, 0x54, + 0x25, 0xd1, 0x2e, 0xe4, 0x26, 0xd1, 0xee, 0x99, 0x13, 0xdb, 0xfe, 0x1e, 0x0b, 0x84, 0x77, 0x64, + 0x1f, 0x67, 0xf7, 0xa7, 0x60, 0x6c, 0x2f, 0x9d, 0x5b, 0xe6, 0x62, 0xbe, 0xbb, 0xa8, 0xc8, 0x28, + 0xa3, 0x78, 0x25, 0x23, 0x8f, 0x8c, 0x41, 0xcb, 0x6e, 0x80, 0x80, 0x96, 0x09, 0x93, 0x98, 0xf6, + 0xee, 0xcd, 0xf3, 0x00, 0x0d, 0x86, 0xab, 0x65, 0x22, 0x57, 0x37, 0x73, 0x59, 0x41, 0xb0, 0x86, + 0x65, 0xff, 0xeb, 0x02, 0x8c, 0xca, 0x5c, 0x26, 0x1d, 0xaf, 0x1f, 0xb9, 0xc6, 0x91, 0x92, 0x1b, + 0xb2, 0xd4, 0xfd, 0x94, 0x70, 0x35, 0x16, 0x07, 0xc5, 0xa9, 0xfb, 0x25, 0x00, 0xc7, 0x38, 0x74, + 0x17, 0x85, 0x9d, 0x4d, 0x86, 0x9e, 0xf0, 0xe5, 0xab, 0xf1, 0x62, 0x2c, 0xe1, 0xe8, 0x13, 0x30, + 0xc5, 0xeb, 0x05, 0x7e, 0xdb, 0xd9, 0xe6, 0xc2, 0xfa, 0x41, 0xe5, 0x84, 0x3f, 0xb5, 0x96, 0x80, + 0x1d, 0x1e, 0xcc, 0x9f, 0x4a, 0x96, 0x31, 0x2d, 0x54, 0x8a, 0x0a, 0xb3, 0xc9, 0xe1, 0x8d, 0xd0, + 0xdd, 0x9f, 0x32, 0xe5, 0x89, 0x41, 0x58, 0xc7, 0xb3, 0x3f, 0x0b, 0x28, 0x9d, 0xd5, 0x05, 0xbd, + 0xce, 0x0d, 0x31, 0xdd, 0x80, 0x34, 0xba, 0x69, 0xa5, 0x74, 0x57, 0x73, 0xe9, 0x86, 0xc3, 0x6b, + 0x61, 0x55, 0xdf, 0xfe, 0xff, 0x8a, 0x30, 0x95, 0x74, 0x3c, 0x46, 0xd7, 0x61, 0x88, 0xb3, 0x1e, + 0x82, 0x7c, 0x17, 0xa3, 0x07, 0xcd, 0x5d, 0x99, 0x1d, 0xc2, 0x82, 0x7b, 0x11, 0xf5, 0xd1, 0x9b, + 0x30, 0xda, 0xf0, 0xef, 0x79, 0xf7, 0x9c, 0xa0, 0xb1, 0x58, 0xad, 0x88, 0xe5, 0x9c, 0xf9, 0x0a, + 0x2b, 0xc7, 0x68, 0xba, 0x0b, 0x34, 0x53, 0xf0, 0xc5, 0x20, 0xac, 0x93, 0x43, 0x1b, 0x2c, 0x08, + 0xf5, 0x96, 0xbb, 0xbd, 0xe6, 0xb4, 0xbb, 0x59, 0xe5, 0x2f, 0x4b, 0x24, 0x8d, 0xf2, 0xb8, 0x88, + 0x54, 0xcd, 0x01, 0x38, 0x26, 0x84, 0x3e, 0x0f, 0x33, 0x61, 0x8e, 0x6c, 0x38, 0x2f, 0xc9, 0x57, + 0x37, 0x71, 0xe9, 0xd2, 0x63, 0xf4, 0x7d, 0x9c, 0x25, 0x45, 0xce, 0x6a, 0xc6, 0xfe, 0x91, 0x53, + 0x60, 0x6c, 0x62, 0x23, 0xe7, 0xa3, 0x75, 0x4c, 0x39, 0x1f, 0x31, 0x8c, 0x90, 0x56, 0x3b, 0xda, + 0x2f, 0xbb, 0x41, 0xb7, 0x4c, 0xc8, 0x2b, 0x02, 0x27, 0x4d, 0x53, 0x42, 0xb0, 0xa2, 0x93, 0x9d, + 0x98, 0xb3, 0xf8, 0x75, 0x4c, 0xcc, 0x39, 0x70, 0x82, 0x89, 0x39, 0xd7, 0x61, 0x78, 0xdb, 0x8d, + 0x30, 0x69, 0xfb, 0x82, 0xe9, 0xcf, 0x5c, 0x87, 0xd7, 0x38, 0x4a, 0x3a, 0x05, 0x9c, 0x00, 0x60, + 0x49, 0x04, 0xbd, 0xae, 0x76, 0xe0, 0x50, 0xfe, 0x9b, 0x39, 0xad, 0x9d, 0xcf, 0xdc, 0x83, 0x22, + 0xfd, 0xe6, 0xf0, 0xc3, 0xa6, 0xdf, 0x5c, 0x95, 0x49, 0x33, 0x47, 0xf2, 0x5d, 0x68, 0x58, 0x4e, + 0xcc, 0x1e, 0xa9, 0x32, 0xef, 0xe8, 0x89, 0x46, 0x4b, 0xf9, 0x27, 0x81, 0xca, 0x21, 0xda, 0x67, + 0x7a, 0xd1, 0xef, 0xb1, 0xe0, 0x74, 0x3b, 0x2b, 0xe7, 0xae, 0x50, 0x64, 0xbf, 0xd4, 0x77, 0x5a, + 0x5f, 0xa3, 0x41, 0x26, 0x00, 0xca, 0x44, 0xc3, 0xd9, 0xcd, 0xd1, 0x81, 0x0e, 0x36, 0x1b, 0x42, + 0xa1, 0x7a, 0x29, 0x27, 0x4f, 0x69, 0x97, 0xec, 0xa4, 0x1b, 0x19, 0x39, 0x31, 0xdf, 0x9f, 0x97, + 0x13, 0xb3, 0xef, 0x4c, 0x98, 0xaf, 0xab, 0x0c, 0xa5, 0xe3, 0xf9, 0x4b, 0x89, 0xe7, 0x1f, 0xed, + 0x99, 0x97, 0xf4, 0x75, 0x95, 0x97, 0xb4, 0x4b, 0x84, 0x51, 0x9e, 0x75, 0xb4, 0x67, 0x36, 0x52, + 0x2d, 0xa3, 0xe8, 0xe4, 0xf1, 0x64, 0x14, 0x35, 0xae, 0x1a, 0x9e, 0xd4, 0xf2, 0xe9, 0x1e, 0x57, + 0x8d, 0x41, 0xb7, 0xfb, 0x65, 0xc3, 0xb3, 0xa7, 0x4e, 0x3f, 0x54, 0xf6, 0xd4, 0x3b, 0x7a, 0x36, + 0x52, 0xd4, 0x23, 0xdd, 0x26, 0x45, 0xea, 0x33, 0x07, 0xe9, 0x1d, 0xfd, 0x02, 0x9c, 0xc9, 0xa7, + 0xab, 0xee, 0xb9, 0x34, 0xdd, 0xcc, 0x2b, 0x30, 0x95, 0xdb, 0xf4, 0xd4, 0xc9, 0xe4, 0x36, 0x3d, + 0x7d, 0xec, 0xb9, 0x4d, 0xcf, 0x9c, 0x40, 0x6e, 0xd3, 0xc7, 0x4e, 0x30, 0xb7, 0xe9, 0x1d, 0x66, + 0xfd, 0xc1, 0x63, 0xcc, 0x88, 0x88, 0xa8, 0xd9, 0xd1, 0x37, 0xb3, 0x02, 0xd1, 0xf0, 0x8f, 0x53, + 0x20, 0x1c, 0x93, 0xca, 0xc8, 0x99, 0x3a, 0xfb, 0x08, 0x72, 0xa6, 0xae, 0xc7, 0x39, 0x53, 0xcf, + 0xe6, 0x4f, 0x75, 0x86, 0xbf, 0x40, 0x4e, 0xa6, 0xd4, 0x3b, 0x7a, 0x86, 0xd3, 0xc7, 0xbb, 0x88, + 0xf8, 0xb3, 0x04, 0x8f, 0x5d, 0xf2, 0x9a, 0xbe, 0xc6, 0xf3, 0x9a, 0x9e, 0xcb, 0x3f, 0xc9, 0x93, + 0xd7, 0x9d, 0x91, 0xcd, 0x94, 0xf6, 0x4b, 0xc5, 0xde, 0x63, 0xb1, 0x5f, 0x73, 0xfa, 0xa5, 0x82, + 0xf7, 0xa5, 0xfb, 0xa5, 0x40, 0x38, 0x26, 0x65, 0x7f, 0x5f, 0x01, 0x2e, 0x74, 0xdf, 0x6f, 0xb1, + 0x34, 0xb5, 0x1a, 0x6b, 0x3c, 0x13, 0xd2, 0x54, 0xfe, 0x66, 0x8b, 0xb1, 0xfa, 0x0e, 0x6b, 0x76, + 0x0d, 0xa6, 0x95, 0xa3, 0x41, 0xd3, 0xad, 0xef, 0xaf, 0xc7, 0x2f, 0x5f, 0xe5, 0x9c, 0x5d, 0x4b, + 0x22, 0xe0, 0x74, 0x1d, 0xb4, 0x08, 0x93, 0x46, 0x61, 0xa5, 0x2c, 0xde, 0x66, 0x4a, 0x7c, 0x5b, + 0x33, 0xc1, 0x38, 0x89, 0x6f, 0x7f, 0xd9, 0x82, 0xc7, 0x72, 0xd2, 0x91, 0xf5, 0x1d, 0xb5, 0x6b, + 0x0b, 0x26, 0xdb, 0x66, 0xd5, 0x1e, 0xc1, 0xfd, 0x8c, 0xa4, 0x67, 0xaa, 0xaf, 0x09, 0x00, 0x4e, + 0x12, 0xb5, 0xff, 0xdc, 0x82, 0xf3, 0x5d, 0x2d, 0xe7, 0x10, 0x86, 0x33, 0xdb, 0xad, 0xd0, 0x59, + 0x0e, 0x48, 0x83, 0x78, 0x91, 0xeb, 0x34, 0x6b, 0x6d, 0x52, 0xd7, 0xe4, 0xe1, 0xcc, 0x04, 0xed, + 0xda, 0x5a, 0x6d, 0x31, 0x8d, 0x81, 0x73, 0x6a, 0xa2, 0x55, 0x40, 0x69, 0x88, 0x98, 0x61, 0x16, + 0x45, 0x38, 0x4d, 0x0f, 0x67, 0xd4, 0x40, 0x1f, 0x81, 0x71, 0x65, 0x91, 0xa7, 0xcd, 0x38, 0x3b, + 0xd8, 0xb1, 0x0e, 0xc0, 0x26, 0xde, 0xd2, 0x95, 0xdf, 0xfc, 0x83, 0x0b, 0xef, 0xfb, 0xed, 0x3f, + 0xb8, 0xf0, 0xbe, 0xdf, 0xfd, 0x83, 0x0b, 0xef, 0xfb, 0x8e, 0x07, 0x17, 0xac, 0xdf, 0x7c, 0x70, + 0xc1, 0xfa, 0xed, 0x07, 0x17, 0xac, 0xdf, 0x7d, 0x70, 0xc1, 0xfa, 0xfd, 0x07, 0x17, 0xac, 0x2f, + 0xfd, 0xe1, 0x85, 0xf7, 0x7d, 0xaa, 0xb0, 0xf7, 0xdc, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x74, + 0x99, 0xb9, 0xe3, 0xac, 0x01, 0x01, 0x00, } func (m *AWSElasticBlockStoreVolumeSource) Marshal() (dAtA []byte, err error) { diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index 8e19c949cff..18e0c398733 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -4750,10 +4750,14 @@ message ServicePort { // +optional optional k8s.io.apimachinery.pkg.util.intstr.IntOrString targetPort = 4; - // The port on each node on which this service is exposed when type=NodePort or LoadBalancer. - // Usually assigned by the system. If specified, it will be allocated to the service - // if unused or else creation of the service will fail. - // Default is to auto-allocate a port if the ServiceType of this Service requires one. + // The port on each node on which this service is exposed when type is + // NodePort or LoadBalancer. Usually assigned by the system. If a value is + // specified, in-range, and not in use it will be used, otherwise the + // operation will fail. If not specified, a port will be allocated if this + // Service requires one. If this field is specified when creating a + // Service which does not need it, creation will fail. This field will be + // wiped when updating a Service to no longer need it (e.g. changing type + // from NodePort to ClusterIP). // More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport // +optional optional int32 nodePort = 5; @@ -4791,41 +4795,68 @@ message ServiceSpec { map selector = 2; // clusterIP is the IP address of the service and is usually assigned - // randomly by the master. If an address is specified manually and is not in - // use by others, it will be allocated to the service; otherwise, creation - // of the service will fail. This field can not be changed through updates. - // Valid values are "None", empty string (""), or a valid IP address. "None" - // can be specified for headless services when proxying is not required. - // Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if - // type is ExternalName. + // randomly. If an address is specified manually, is in-range (as per + // system configuration), and is not in use, it will be allocated to the + // service; otherwise creation of the service will fail. This field may not + // be changed through updates unless the type field is also being changed + // to ExternalName (which requires this field to be blank) or the type + // field is being changed from ExternalName (in which case this field may + // optionally be specified, as describe above). Valid values are "None", + // empty string (""), or a valid IP address. Setting this to "None" makes a + // "headless service" (no virtual IP), which is useful when direct endpoint + // connections are preferred and proxying is not required. Only applies to + // types ClusterIP, NodePort, and LoadBalancer. If this field is specified + // when creating a Service of type ExternalName, creation will fail. This + // field will be wiped when updating a Service to type ExternalName. // More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies // +optional optional string clusterIP = 3; - // ClusterIPs identifies all the ClusterIPs assigned to this - // service. ClusterIPs are assigned or reserved based on the values of - // service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are - // allowed in ClusterIPs. The IPFamily of each ClusterIP must match - // values provided in service.spec.ipFamilies. Clients using ClusterIPs must - // keep it in sync with ClusterIP (if provided) by having ClusterIP matching - // first element of ClusterIPs. + // ClusterIPs is a list of IP addresses assigned to this service, and are + // usually assigned randomly. If an address is specified manually, is + // in-range (as per system configuration), and is not in use, it will be + // allocated to the service; otherwise creation of the service will fail. + // This field may not be changed through updates unless the type field is + // also being changed to ExternalName (which requires this field to be + // empty) or the type field is being changed from ExternalName (in which + // case this field may optionally be specified, as describe above). Valid + // values are "None", empty string (""), or a valid IP address. Setting + // this to "None" makes a "headless service" (no virtual IP), which is + // useful when direct endpoint connections are preferred and proxying is + // not required. Only applies to types ClusterIP, NodePort, and + // LoadBalancer. If this field is specified when creating a Service of type + // ExternalName, creation will fail. This field will be wiped when updating + // a Service to type ExternalName. If this field is not specified, it will + // be initialized from the clusterIP field. If this field is specified, + // clients must ensure that clusterIPs[0] and clusterIP have the same + // value. + // + // Unless the "IPv6DualStack" feature gate is enabled, this field is + // limited to one value, which must be the same as the clusterIP field. If + // the feature gate is enabled, this field may hold a maximum of two + // entries (dual-stack IPs, in either order). These IPs must correspond to + // the values of the ipFamilies field. Both clusterIPs and ipFamilies are + // governed by the ipFamilyPolicy field. + // More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies // +listType=atomic // +optional repeated string clusterIPs = 18; // type determines how the Service is exposed. Defaults to ClusterIP. Valid // options are ExternalName, ClusterIP, NodePort, and LoadBalancer. - // "ExternalName" maps to the specified externalName. - // "ClusterIP" allocates a cluster-internal IP address for load-balancing to - // endpoints. Endpoints are determined by the selector or if that is not - // specified, by manual construction of an Endpoints object. If clusterIP is - // "None", no virtual IP is allocated and the endpoints are published as a - // set of endpoints rather than a stable IP. + // "ClusterIP" allocates a cluster-internal IP address for load-balancing + // to endpoints. Endpoints are determined by the selector or if that is not + // specified, by manual construction of an Endpoints object or + // EndpointSlice objects. If clusterIP is "None", no virtual IP is + // allocated and the endpoints are published as a set of endpoints rather + // than a virtual IP. // "NodePort" builds on ClusterIP and allocates a port on every node which - // routes to the clusterIP. - // "LoadBalancer" builds on NodePort and creates an - // external load-balancer (if supported in the current cloud) which routes - // to the clusterIP. + // routes to the same endpoints as the clusterIP. + // "LoadBalancer" builds on NodePort and creates an external load-balancer + // (if supported in the current cloud) which routes to the same endpoints + // as the clusterIP. + // "ExternalName" aliases this service to the specified externalName. + // Several other fields do not apply to ExternalName services. // More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types // +optional optional string type = 4; @@ -4861,10 +4892,10 @@ message ServiceSpec { // +optional repeated string loadBalancerSourceRanges = 9; - // externalName is the external reference that kubedns or equivalent will - // return as a CNAME record for this service. No proxying will be involved. - // Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) - // and requires Type to be ExternalName. + // externalName is the external reference that discovery mechanisms will + // return as an alias for this service (e.g. a DNS CNAME record). No + // proxying will be involved. Must be a lowercase RFC-1123 hostname + // (https://tools.ietf.org/html/rfc1123) and requires Type to be // +optional optional string externalName = 10; @@ -4878,10 +4909,14 @@ message ServiceSpec { optional string externalTrafficPolicy = 11; // healthCheckNodePort specifies the healthcheck nodePort for the service. - // If not specified, HealthCheckNodePort is created by the service api - // backend with the allocated nodePort. Will use user-specified nodePort value - // if specified by the client. Only effects when Type is set to LoadBalancer - // and ExternalTrafficPolicy is set to Local. + // This only applies when type is set to LoadBalancer and + // externalTrafficPolicy is set to Local. If a value is specified, is + // in-range, and is not in use, it will be used. If not specified, a value + // will be automatically allocated. External systems (e.g. load-balancers) + // can use this port to determine if a given node holds endpoints for this + // service or not. If this field is specified when creating a Service + // which does not need it, creation will fail. This field will be wiped + // when updating a Service to no longer need it (e.g. changing type). // +optional optional int32 healthCheckNodePort = 12; @@ -4900,16 +4935,6 @@ message ServiceSpec { // +optional optional SessionAffinityConfig sessionAffinityConfig = 14; - // IPFamilies identifies all the IPFamilies assigned for this Service. If a value - // was not provided for IPFamilies it will be defaulted based on the cluster - // configuration and the value of service.spec.ipFamilyPolicy. A maximum of two - // values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is - // conditionally mutable: it allows for adding or removing a secondary IPFamily, - // but it does not allow changing the primary IPFamily of the service. - // +listType=atomic - // +optional - repeated string ipFamilies = 19; - // topologyKeys is a preference-order list of topology keys which // implementations of services should use to preferentially sort endpoints // when accessing this Service, it can not be used at the same time as @@ -4925,14 +4950,36 @@ message ServiceSpec { // +optional repeated string topologyKeys = 16; - // IPFamilyPolicy represents the dual-stack-ness requested or required by this - // Service. If there is no value provided, then this Service will be considered - // SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), - // PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single - // IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies - // on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned - // to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs - // respectively. + // IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + // service, and is gated by the "IPv6DualStack" feature gate. This field + // is usually assigned automatically based on cluster configuration and the + // ipFamilyPolicy field. If this field is specified manually, the requested + // family is available in the cluster, and ipFamilyPolicy allows it, it + // will be used; otherwise creation of the service will fail. This field + // is conditionally mutable: it allows for adding or removing a secondary + // IP family, but it does not allow changing the primary IP family of the + // Service. Valid values are "IPv4" and "IPv6". This field only applies + // to Services of types ClusterIP, NodePort, and LoadBalancer, and does + // apply to "headless" services. This field will be wiped when updating a + // Service to type ExternalName. + // + // This field may hold a maximum of two entries (dual-stack families, in + // either order). These families must correspond to the values of the + // clusterIPs field, if specified. Both clusterIPs and ipFamilies are + // governed by the ipFamilyPolicy field. + // +listType=atomic + // +optional + repeated string ipFamilies = 19; + + // IPFamilyPolicy represents the dual-stack-ness requested or required by + // this Service, and is gated by the "IPv6DualStack" feature gate. If + // there is no value provided, then this field will be set to SingleStack. + // Services can be "SingleStack" (a single IP family), "PreferDualStack" + // (two IP families on dual-stack configured clusters or a single IP family + // on single-stack clusters), or "RequireDualStack" (two IP families on + // dual-stack configured clusters, otherwise fail). The ipFamilies and + // clusterIPs fields depend on the value of this field. This field will be + // wiped when updating a service to type ExternalName. // +optional optional string ipFamilyPolicy = 17; } diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 1e21f17328c..de90357b476 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4033,41 +4033,68 @@ type ServiceSpec struct { Selector map[string]string `json:"selector,omitempty" protobuf:"bytes,2,rep,name=selector"` // clusterIP is the IP address of the service and is usually assigned - // randomly by the master. If an address is specified manually and is not in - // use by others, it will be allocated to the service; otherwise, creation - // of the service will fail. This field can not be changed through updates. - // Valid values are "None", empty string (""), or a valid IP address. "None" - // can be specified for headless services when proxying is not required. - // Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if - // type is ExternalName. + // randomly. If an address is specified manually, is in-range (as per + // system configuration), and is not in use, it will be allocated to the + // service; otherwise creation of the service will fail. This field may not + // be changed through updates unless the type field is also being changed + // to ExternalName (which requires this field to be blank) or the type + // field is being changed from ExternalName (in which case this field may + // optionally be specified, as describe above). Valid values are "None", + // empty string (""), or a valid IP address. Setting this to "None" makes a + // "headless service" (no virtual IP), which is useful when direct endpoint + // connections are preferred and proxying is not required. Only applies to + // types ClusterIP, NodePort, and LoadBalancer. If this field is specified + // when creating a Service of type ExternalName, creation will fail. This + // field will be wiped when updating a Service to type ExternalName. // More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies // +optional ClusterIP string `json:"clusterIP,omitempty" protobuf:"bytes,3,opt,name=clusterIP"` - // ClusterIPs identifies all the ClusterIPs assigned to this - // service. ClusterIPs are assigned or reserved based on the values of - // service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are - // allowed in ClusterIPs. The IPFamily of each ClusterIP must match - // values provided in service.spec.ipFamilies. Clients using ClusterIPs must - // keep it in sync with ClusterIP (if provided) by having ClusterIP matching - // first element of ClusterIPs. + // ClusterIPs is a list of IP addresses assigned to this service, and are + // usually assigned randomly. If an address is specified manually, is + // in-range (as per system configuration), and is not in use, it will be + // allocated to the service; otherwise creation of the service will fail. + // This field may not be changed through updates unless the type field is + // also being changed to ExternalName (which requires this field to be + // empty) or the type field is being changed from ExternalName (in which + // case this field may optionally be specified, as describe above). Valid + // values are "None", empty string (""), or a valid IP address. Setting + // this to "None" makes a "headless service" (no virtual IP), which is + // useful when direct endpoint connections are preferred and proxying is + // not required. Only applies to types ClusterIP, NodePort, and + // LoadBalancer. If this field is specified when creating a Service of type + // ExternalName, creation will fail. This field will be wiped when updating + // a Service to type ExternalName. If this field is not specified, it will + // be initialized from the clusterIP field. If this field is specified, + // clients must ensure that clusterIPs[0] and clusterIP have the same + // value. + // + // Unless the "IPv6DualStack" feature gate is enabled, this field is + // limited to one value, which must be the same as the clusterIP field. If + // the feature gate is enabled, this field may hold a maximum of two + // entries (dual-stack IPs, in either order). These IPs must correspond to + // the values of the ipFamilies field. Both clusterIPs and ipFamilies are + // governed by the ipFamilyPolicy field. + // More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies // +listType=atomic // +optional ClusterIPs []string `json:"clusterIPs,omitempty" protobuf:"bytes,18,opt,name=clusterIPs"` // type determines how the Service is exposed. Defaults to ClusterIP. Valid // options are ExternalName, ClusterIP, NodePort, and LoadBalancer. - // "ExternalName" maps to the specified externalName. - // "ClusterIP" allocates a cluster-internal IP address for load-balancing to - // endpoints. Endpoints are determined by the selector or if that is not - // specified, by manual construction of an Endpoints object. If clusterIP is - // "None", no virtual IP is allocated and the endpoints are published as a - // set of endpoints rather than a stable IP. + // "ClusterIP" allocates a cluster-internal IP address for load-balancing + // to endpoints. Endpoints are determined by the selector or if that is not + // specified, by manual construction of an Endpoints object or + // EndpointSlice objects. If clusterIP is "None", no virtual IP is + // allocated and the endpoints are published as a set of endpoints rather + // than a virtual IP. // "NodePort" builds on ClusterIP and allocates a port on every node which - // routes to the clusterIP. - // "LoadBalancer" builds on NodePort and creates an - // external load-balancer (if supported in the current cloud) which routes - // to the clusterIP. + // routes to the same endpoints as the clusterIP. + // "LoadBalancer" builds on NodePort and creates an external load-balancer + // (if supported in the current cloud) which routes to the same endpoints + // as the clusterIP. + // "ExternalName" aliases this service to the specified externalName. + // Several other fields do not apply to ExternalName services. // More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types // +optional Type ServiceType `json:"type,omitempty" protobuf:"bytes,4,opt,name=type,casttype=ServiceType"` @@ -4103,10 +4130,10 @@ type ServiceSpec struct { // +optional LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty" protobuf:"bytes,9,opt,name=loadBalancerSourceRanges"` - // externalName is the external reference that kubedns or equivalent will - // return as a CNAME record for this service. No proxying will be involved. - // Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) - // and requires Type to be ExternalName. + // externalName is the external reference that discovery mechanisms will + // return as an alias for this service (e.g. a DNS CNAME record). No + // proxying will be involved. Must be a lowercase RFC-1123 hostname + // (https://tools.ietf.org/html/rfc1123) and requires Type to be // +optional ExternalName string `json:"externalName,omitempty" protobuf:"bytes,10,opt,name=externalName"` @@ -4120,10 +4147,14 @@ type ServiceSpec struct { ExternalTrafficPolicy ServiceExternalTrafficPolicyType `json:"externalTrafficPolicy,omitempty" protobuf:"bytes,11,opt,name=externalTrafficPolicy"` // healthCheckNodePort specifies the healthcheck nodePort for the service. - // If not specified, HealthCheckNodePort is created by the service api - // backend with the allocated nodePort. Will use user-specified nodePort value - // if specified by the client. Only effects when Type is set to LoadBalancer - // and ExternalTrafficPolicy is set to Local. + // This only applies when type is set to LoadBalancer and + // externalTrafficPolicy is set to Local. If a value is specified, is + // in-range, and is not in use, it will be used. If not specified, a value + // will be automatically allocated. External systems (e.g. load-balancers) + // can use this port to determine if a given node holds endpoints for this + // service or not. If this field is specified when creating a Service + // which does not need it, creation will fail. This field will be wiped + // when updating a Service to no longer need it (e.g. changing type). // +optional HealthCheckNodePort int32 `json:"healthCheckNodePort,omitempty" protobuf:"bytes,12,opt,name=healthCheckNodePort"` @@ -4142,19 +4173,6 @@ type ServiceSpec struct { // +optional SessionAffinityConfig *SessionAffinityConfig `json:"sessionAffinityConfig,omitempty" protobuf:"bytes,14,opt,name=sessionAffinityConfig"` - // IPFamily is tombstoned to show why 15 is a reserved protobuf tag. - // IPFamily *IPFamily `json:"ipFamily,omitempty" protobuf:"bytes,15,opt,name=ipFamily,Configcasttype=IPFamily"` - - // IPFamilies identifies all the IPFamilies assigned for this Service. If a value - // was not provided for IPFamilies it will be defaulted based on the cluster - // configuration and the value of service.spec.ipFamilyPolicy. A maximum of two - // values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is - // conditionally mutable: it allows for adding or removing a secondary IPFamily, - // but it does not allow changing the primary IPFamily of the service. - // +listType=atomic - // +optional - IPFamilies []IPFamily `json:"ipFamilies,omitempty" protobuf:"bytes,19,opt,name=ipFamilies,casttype=IPFamily"` - // topologyKeys is a preference-order list of topology keys which // implementations of services should use to preferentially sort endpoints // when accessing this Service, it can not be used at the same time as @@ -4170,14 +4188,39 @@ type ServiceSpec struct { // +optional TopologyKeys []string `json:"topologyKeys,omitempty" protobuf:"bytes,16,opt,name=topologyKeys"` - // IPFamilyPolicy represents the dual-stack-ness requested or required by this - // Service. If there is no value provided, then this Service will be considered - // SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), - // PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single - // IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies - // on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned - // to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs - // respectively. + // IPFamily is tombstoned to show why 15 is a reserved protobuf tag. + // IPFamily *IPFamily `json:"ipFamily,omitempty" protobuf:"bytes,15,opt,name=ipFamily,Configcasttype=IPFamily"` + + // IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + // service, and is gated by the "IPv6DualStack" feature gate. This field + // is usually assigned automatically based on cluster configuration and the + // ipFamilyPolicy field. If this field is specified manually, the requested + // family is available in the cluster, and ipFamilyPolicy allows it, it + // will be used; otherwise creation of the service will fail. This field + // is conditionally mutable: it allows for adding or removing a secondary + // IP family, but it does not allow changing the primary IP family of the + // Service. Valid values are "IPv4" and "IPv6". This field only applies + // to Services of types ClusterIP, NodePort, and LoadBalancer, and does + // apply to "headless" services. This field will be wiped when updating a + // Service to type ExternalName. + // + // This field may hold a maximum of two entries (dual-stack families, in + // either order). These families must correspond to the values of the + // clusterIPs field, if specified. Both clusterIPs and ipFamilies are + // governed by the ipFamilyPolicy field. + // +listType=atomic + // +optional + IPFamilies []IPFamily `json:"ipFamilies,omitempty" protobuf:"bytes,19,opt,name=ipFamilies,casttype=IPFamily"` + + // IPFamilyPolicy represents the dual-stack-ness requested or required by + // this Service, and is gated by the "IPv6DualStack" feature gate. If + // there is no value provided, then this field will be set to SingleStack. + // Services can be "SingleStack" (a single IP family), "PreferDualStack" + // (two IP families on dual-stack configured clusters or a single IP family + // on single-stack clusters), or "RequireDualStack" (two IP families on + // dual-stack configured clusters, otherwise fail). The ipFamilies and + // clusterIPs fields depend on the value of this field. This field will be + // wiped when updating a service to type ExternalName. // +optional IPFamilyPolicy *IPFamilyPolicyType `json:"ipFamilyPolicy,omitempty" protobuf:"bytes,17,opt,name=ipFamilyPolicy,casttype=IPFamilyPolicyType"` } @@ -4222,10 +4265,14 @@ type ServicePort struct { // +optional TargetPort intstr.IntOrString `json:"targetPort,omitempty" protobuf:"bytes,4,opt,name=targetPort"` - // The port on each node on which this service is exposed when type=NodePort or LoadBalancer. - // Usually assigned by the system. If specified, it will be allocated to the service - // if unused or else creation of the service will fail. - // Default is to auto-allocate a port if the ServiceType of this Service requires one. + // The port on each node on which this service is exposed when type is + // NodePort or LoadBalancer. Usually assigned by the system. If a value is + // specified, in-range, and not in use it will be used, otherwise the + // operation will fail. If not specified, a port will be allocated if this + // Service requires one. If this field is specified when creating a + // Service which does not need it, creation will fail. This field will be + // wiped when updating a Service to no longer need it (e.g. changing type + // from NodePort to ClusterIP). // More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport // +optional NodePort int32 `json:"nodePort,omitempty" protobuf:"varint,5,opt,name=nodePort"` diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index a71dc118f24..a507ac28875 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -2209,7 +2209,7 @@ var map_ServicePort = map[string]string{ "appProtocol": "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. This is a beta field that is guarded by the ServiceAppProtocol feature gate and enabled by default.", "port": "The port that will be exposed by this service.", "targetPort": "Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod's container ports. If this is not specified, the value of the 'port' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the 'port' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service", - "nodePort": "The port on each node on which this service is exposed when type=NodePort or LoadBalancer. Usually assigned by the system. If specified, it will be allocated to the service if unused or else creation of the service will fail. Default is to auto-allocate a port if the ServiceType of this Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", + "nodePort": "The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not specified, a port will be allocated if this Service requires one. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport", } func (ServicePort) SwaggerDoc() map[string]string { @@ -2229,21 +2229,21 @@ var map_ServiceSpec = map[string]string{ "": "ServiceSpec describes the attributes that a user creates on a service.", "ports": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "selector": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", - "clusterIP": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", - "clusterIPs": "ClusterIPs identifies all the ClusterIPs assigned to this service. ClusterIPs are assigned or reserved based on the values of service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are allowed in ClusterIPs. The IPFamily of each ClusterIP must match values provided in service.spec.ipFamilies. Clients using ClusterIPs must keep it in sync with ClusterIP (if provided) by having ClusterIP matching first element of ClusterIPs.", - "type": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", + "clusterIP": "clusterIP is the IP address of the service and is usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be blank) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", + "clusterIPs": "ClusterIPs is a list of IP addresses assigned to this service, and are usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be empty) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. If this field is not specified, it will be initialized from the clusterIP field. If this field is specified, clients must ensure that clusterIPs[0] and clusterIP have the same value.\n\nUnless the \"IPv6DualStack\" feature gate is enabled, this field is limited to one value, which must be the same as the clusterIP field. If the feature gate is enabled, this field may hold a maximum of two entries (dual-stack IPs, in either order). These IPs must correspond to the values of the ipFamilies field. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", + "type": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object or EndpointSlice objects. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a virtual IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the same endpoints as the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the same endpoints as the clusterIP. \"ExternalName\" aliases this service to the specified externalName. Several other fields do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "externalIPs": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "sessionAffinity": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "loadBalancerIP": "Only applies to Service Type: LoadBalancer LoadBalancer will get created with the IP specified in this field. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature.", "loadBalancerSourceRanges": "If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/", - "externalName": "externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be ExternalName.", + "externalName": "externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires Type to be", "externalTrafficPolicy": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", - "healthCheckNodePort": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", + "healthCheckNodePort": "healthCheckNodePort specifies the healthcheck nodePort for the service. This only applies when type is set to LoadBalancer and externalTrafficPolicy is set to Local. If a value is specified, is in-range, and is not in use, it will be used. If not specified, a value will be automatically allocated. External systems (e.g. load-balancers) can use this port to determine if a given node holds endpoints for this service or not. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type).", "publishNotReadyAddresses": "publishNotReadyAddresses indicates that any agent which deals with endpoints for this Service should disregard any indications of ready/not-ready. The primary use case for setting this field is for a StatefulSet's Headless Service to propagate SRV DNS records for its Pods for the purpose of peer discovery. The Kubernetes controllers that generate Endpoints and EndpointSlice resources for Services interpret this to mean that all endpoints are considered \"ready\" even if the Pods themselves are not. Agents which consume only Kubernetes generated endpoints through the Endpoints or EndpointSlice resources can safely assume this behavior.", "sessionAffinityConfig": "sessionAffinityConfig contains the configurations of session affinity.", - "ipFamilies": "IPFamilies identifies all the IPFamilies assigned for this Service. If a value was not provided for IPFamilies it will be defaulted based on the cluster configuration and the value of service.spec.ipFamilyPolicy. A maximum of two values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is conditionally mutable: it allows for adding or removing a secondary IPFamily, but it does not allow changing the primary IPFamily of the service.", "topologyKeys": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", - "ipFamilyPolicy": "IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this Service will be considered SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs respectively.", + "ipFamilies": "IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this service, and is gated by the \"IPv6DualStack\" feature gate. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply to \"headless\" services. This field will be wiped when updating a Service to type ExternalName.\n\nThis field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field.", + "ipFamilyPolicy": "IPFamilyPolicy represents the dual-stack-ness requested or required by this Service, and is gated by the \"IPv6DualStack\" feature gate. If there is no value provided, then this field will be set to SingleStack. Services can be \"SingleStack\" (a single IP family), \"PreferDualStack\" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or \"RequireDualStack\" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName.", } func (ServiceSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go index 2b070041496..307e30440b0 100644 --- a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go @@ -5290,16 +5290,16 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = new(SessionAffinityConfig) (*in).DeepCopyInto(*out) } - if in.IPFamilies != nil { - in, out := &in.IPFamilies, &out.IPFamilies - *out = make([]IPFamily, len(*in)) - copy(*out, *in) - } if in.TopologyKeys != nil { in, out := &in.TopologyKeys, &out.TopologyKeys *out = make([]string, len(*in)) copy(*out, *in) } + if in.IPFamilies != nil { + in, out := &in.IPFamilies, &out.IPFamilies + *out = make([]IPFamily, len(*in)) + copy(*out, *in) + } if in.IPFamilyPolicy != nil { in, out := &in.IPFamilyPolicy, &out.IPFamilyPolicy *out = new(IPFamilyPolicyType) diff --git a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json index 338d9936fab..360cfe645be 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json +++ b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json @@ -76,13 +76,13 @@ "timeoutSeconds": 2072604405 } }, - "ipFamilies": [ - "x" - ], "topologyKeys": [ "30" ], - "ipFamilyPolicy": ";Ơ歿:狞夌碕ʂ" + "ipFamilies": [ + "斬³;Ơ歿:狞夌碕ʂɭîcP$Iņɖ" + ], + "ipFamilyPolicy": "9ȫŚ" }, "status": { "loadBalancer": { diff --git a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.pb b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.pb index 3993d0a896e88962ad16d6558c1f3e93828f1f41..68975591f6c92eab2288c500bbb92039c1e84e22 100644 GIT binary patch delta 84 zcmV-a0IUC_1Goc_6ak)*6|Mo{k;Jh>1v$v8#hQ`<0x~t40VC#?tirQ9#-Qe{zdGlO qp5>&B=c1L$g2}DJu47OnNyUcAmKqES3JM4c0y8lZ0y8oi03rZ)Y9na? delta 63 zcmV-F0Kosa1ET|w6ak5m6|Moyk;Jhs5Xv=Zl`@q>SgHmCAyW0Rl2LngIcL V8Vm{w3J3}UGcghZGcp