diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 60ed4f71de4..c78c7543c77 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -10257,6 +10257,14 @@ "description": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, + "clusterIPs": { + "description": "ClusterIPs identifies all the ClusterIPs assigned to this service. ClusterIPs are assigned or reserved based on the values of service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are allowed in ClusterIPs. The IPFamily of each ClusterIP must match values provided in service.spec.ipFamilies. Clients using ClusterIPs must keep it in sync with ClusterIP (if provided) by having ClusterIP matching first element of ClusterIPs.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, "externalIPs": { "description": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "items": { @@ -10277,8 +10285,16 @@ "format": "int32", "type": "integer" }, - "ipFamily": { - "description": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6) when the IPv6DualStack feature gate is enabled. In a dual-stack cluster, you can specify ipFamily when creating a ClusterIP Service to determine whether the controller will allocate an IPv4 or IPv6 IP for it, and you can specify ipFamily when creating a headless Service to determine whether it will have IPv4 or IPv6 Endpoints. In either case, if you do not specify an ipFamily explicitly, it will default to the cluster's primary IP family. This field is part of an alpha feature, and you should not make any assumptions about its semantics other than those described above. In particular, you should not assume that it can (or cannot) be changed after creation time; that it can only have the values \"IPv4\" and \"IPv6\"; or that its current value on a given Service correctly reflects the current state of that Service. (For ClusterIP Services, look at clusterIP to see if the Service is IPv4 or IPv6. For headless Services, look at the endpoints, which may be dual-stack in the future. For ExternalName Services, ipFamily has no meaning, but it may be set to an irrelevant value anyway.)", + "ipFamilies": { + "description": "IPFamilies identifies all the IPFamilies assigned for this Service. If a value was not provided for IPFamilies it will be defaulted based on the cluster configuration and the value of service.spec.ipFamilyPolicy. A maximum of two values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is conditionally mutable: it allows for adding or removing a secondary IPFamily, but it does not allow changing the primary IPFamily of the service.", + "items": { + "type": "string" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + }, + "ipFamilyPolicy": { + "description": "IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this Service will be considered SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs respectively.", "type": "string" }, "loadBalancerIP": { diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index 25519a67832..88cba4558be 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -55,9 +55,12 @@ func validateClusterIPFlags(options *ServerRunOptions) []error { } // Secondary IP validation + // while api-server dualstack bits does not have dependency on EndPointSlice, its + // a good idea to have validation consistent across all components (ControllerManager + // needs EndPointSlice + DualStack feature flags). secondaryServiceClusterIPRangeUsed := (options.SecondaryServiceClusterIPRange.IP != nil) - if secondaryServiceClusterIPRangeUsed && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - errs = append(errs, fmt.Errorf("--secondary-service-cluster-ip-range can only be used if %v feature is enabled", string(features.IPv6DualStack))) + if secondaryServiceClusterIPRangeUsed && (!utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice)) { + errs = append(errs, fmt.Errorf("secondary service cluster-ip range(--service-cluster-ip-range[1]) can only be used if %v and %v feature is enabled", string(features.IPv6DualStack), string(features.EndpointSlice))) } // note: While the cluster might be dualstack (i.e. pods with multiple IPs), the user may choose @@ -68,14 +71,14 @@ func validateClusterIPFlags(options *ServerRunOptions) []error { // Should be dualstack IPFamily(PrimaryServiceClusterIPRange) != IPFamily(SecondaryServiceClusterIPRange) dualstack, err := netutils.IsDualStackCIDRs([]*net.IPNet{&options.PrimaryServiceClusterIPRange, &options.SecondaryServiceClusterIPRange}) if err != nil { - errs = append(errs, errors.New("error attempting to validate dualstack for --service-cluster-ip-range and --secondary-service-cluster-ip-range")) + errs = append(errs, fmt.Errorf("error attempting to validate dualstack for --service-cluster-ip-range value error:%v", err)) } if !dualstack { - errs = append(errs, errors.New("--service-cluster-ip-range and --secondary-service-cluster-ip-range must be of different IP family")) + errs = append(errs, errors.New("--service-cluster-ip-range[0] and --service-cluster-ip-range[1] must be of different IP family")) } - if err := validateMaxCIDRRange(options.SecondaryServiceClusterIPRange, maxCIDRBits, "--secondary-service-cluster-ip-range"); err != nil { + if err := validateMaxCIDRRange(options.SecondaryServiceClusterIPRange, maxCIDRBits, "--service-cluster-ip-range[1]"); err != nil { errs = append(errs, err) } } diff --git a/cmd/kube-apiserver/app/options/validation_test.go b/cmd/kube-apiserver/app/options/validation_test.go index 73e66b58a1f..24aa1fa972d 100644 --- a/cmd/kube-apiserver/app/options/validation_test.go +++ b/cmd/kube-apiserver/app/options/validation_test.go @@ -54,10 +54,11 @@ func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *Serv func TestClusterSerivceIPRange(t *testing.T) { testCases := []struct { - name string - options *ServerRunOptions - enableDualStack bool - expectErrors bool + name string + options *ServerRunOptions + enableDualStack bool + enableEndpointSlice bool + expectErrors bool }{ { name: "no service cidr", @@ -66,10 +67,11 @@ func TestClusterSerivceIPRange(t *testing.T) { enableDualStack: false, }, { - name: "only secondary service cidr, dual stack gate on", - expectErrors: true, - options: makeOptionsWithCIDRs("", "10.0.0.0/16"), - enableDualStack: true, + name: "only secondary service cidr, dual stack gate on", + expectErrors: true, + options: makeOptionsWithCIDRs("", "10.0.0.0/16"), + enableDualStack: true, + enableEndpointSlice: true, }, { name: "only secondary service cidr, dual stack gate off", @@ -78,16 +80,18 @@ func TestClusterSerivceIPRange(t *testing.T) { enableDualStack: false, }, { - name: "primary and secondary are provided but not dual stack v4-v4", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/16", "11.0.0.0/16"), - enableDualStack: true, + name: "primary and secondary are provided but not dual stack v4-v4", + expectErrors: true, + options: makeOptionsWithCIDRs("10.0.0.0/16", "11.0.0.0/16"), + enableDualStack: true, + enableEndpointSlice: true, }, { - name: "primary and secondary are provided but not dual stack v6-v6", - expectErrors: true, - options: makeOptionsWithCIDRs("2000::/108", "3000::/108"), - enableDualStack: true, + name: "primary and secondary are provided but not dual stack v6-v6", + expectErrors: true, + options: makeOptionsWithCIDRs("2000::/108", "3000::/108"), + enableDualStack: true, + enableEndpointSlice: true, }, { name: "valid dual stack with gate disabled", @@ -96,16 +100,33 @@ func TestClusterSerivceIPRange(t *testing.T) { enableDualStack: false, }, { - name: "service cidr to big", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/8", ""), - enableDualStack: true, + name: "service cidr is too big", + expectErrors: true, + options: makeOptionsWithCIDRs("10.0.0.0/8", ""), + enableDualStack: true, + enableEndpointSlice: true, }, { - name: "dual-stack secondary cidr to big", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64"), - enableDualStack: true, + name: "dual-stack secondary cidr too big", + expectErrors: true, + options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64"), + enableDualStack: true, + enableEndpointSlice: true, + }, + { + name: "valid v6-v4 dual stack + gate on + endpointSlice gate is on", + expectErrors: false, + options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"), + enableDualStack: true, + enableEndpointSlice: true, + }, + + { + name: "valid v4-v6 dual stack + gate on + endpointSlice is off", + expectErrors: true, + options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"), + enableDualStack: true, + enableEndpointSlice: false, }, /* success cases */ { @@ -115,22 +136,25 @@ func TestClusterSerivceIPRange(t *testing.T) { enableDualStack: false, }, { - name: "valid v4-v6 dual stack + gate on", - expectErrors: false, - options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"), - enableDualStack: true, + name: "valid v4-v6 dual stack + gate on", + expectErrors: false, + options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"), + enableDualStack: true, + enableEndpointSlice: true, }, { - name: "valid v6-v4 dual stack + gate on", - expectErrors: false, - options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"), - enableDualStack: true, + name: "valid v6-v4 dual stack + gate on", + expectErrors: false, + options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"), + enableDualStack: true, + enableEndpointSlice: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EndpointSlice, tc.enableEndpointSlice)() errs := validateClusterIPFlags(tc.options) if len(errs) > 0 && !tc.expectErrors { t.Errorf("expected no errors, errors found %+v", errs) diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index b17a186814f..4c9a841f5ae 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -110,14 +110,14 @@ func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error) return nil, false, err } - // failure: more than one cidr and dual stack is not enabled - if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs)) + // failure: more than one cidr and dual stack is not enabled and/or endpoint slice is not enabled + if len(clusterCIDRs) > 1 && (!utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice)) { + return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack or EndpointSlice feature is not enabled", len(clusterCIDRs)) } // failure: more than one cidr but they are not configured as dual stack if len(clusterCIDRs) > 1 && !dualStack { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs)) + return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily)", len(clusterCIDRs)) } // failure: more than cidrs is not allowed even with dual stack diff --git a/pkg/apis/core/helper/helpers.go b/pkg/apis/core/helper/helpers.go index e93803dd1d4..1ed2f59facd 100644 --- a/pkg/apis/core/helper/helpers.go +++ b/pkg/apis/core/helper/helpers.go @@ -267,7 +267,10 @@ func IsIntegerResourceName(str string) bool { // IsServiceIPSet aims to check if the service's ClusterIP is set or not // the objective is not to perform validation here func IsServiceIPSet(service *core.Service) bool { - return service.Spec.ClusterIP != core.ClusterIPNone && service.Spec.ClusterIP != "" + // This function assumes that the service is semantically validated + // it does not test if the IP is valid, just makes sure that it is set. + return len(service.Spec.ClusterIP) > 0 && + service.Spec.ClusterIP != core.ClusterIPNone } var standardFinalizers = sets.NewString( diff --git a/pkg/apis/core/helper/helpers_test.go b/pkg/apis/core/helper/helpers_test.go index 9eac295fdfd..0d197651cfc 100644 --- a/pkg/apis/core/helper/helpers_test.go +++ b/pkg/apis/core/helper/helpers_test.go @@ -294,3 +294,73 @@ func TestIsOvercommitAllowed(t *testing.T) { } } } + +func TestIsServiceIPSet(t *testing.T) { + testCases := []struct { + input core.ServiceSpec + output bool + name string + }{ + { + name: "nil cluster ip", + input: core.ServiceSpec{ + ClusterIPs: nil, + }, + + output: false, + }, + { + name: "headless service", + input: core.ServiceSpec{ + ClusterIP: "None", + ClusterIPs: []string{"None"}, + }, + output: false, + }, + // true cases + { + name: "one ipv4", + input: core.ServiceSpec{ + ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, + }, + output: true, + }, + { + name: "one ipv6", + input: core.ServiceSpec{ + ClusterIP: "2001::1", + ClusterIPs: []string{"2001::1"}, + }, + output: true, + }, + { + name: "v4, v6", + input: core.ServiceSpec{ + ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4", "2001::1"}, + }, + output: true, + }, + { + name: "v6, v4", + input: core.ServiceSpec{ + ClusterIP: "2001::1", + ClusterIPs: []string{"2001::1", "1.2.3.4"}, + }, + + output: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := core.Service{ + Spec: tc.input, + } + if IsServiceIPSet(&s) != tc.output { + t.Errorf("case, input: %v, expected: %v, got: %v", tc.input, tc.output, !tc.output) + } + }) + } +} diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 87805bfecf5..0c6baa98332 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -3510,8 +3510,13 @@ type LoadBalancerIngress struct { Hostname string } +const ( + // MaxServiceTopologyKeys is the largest number of topology keys allowed on a service + MaxServiceTopologyKeys = 16 +) + // IPFamily represents the IP Family (IPv4 or IPv6). This type is used -// to express the family of an IP expressed by a type (i.e. service.Spec.IPFamily) +// to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). type IPFamily string const ( @@ -3519,8 +3524,29 @@ const ( IPv4Protocol IPFamily = "IPv4" // IPv6Protocol indicates that this IP is IPv6 protocol IPv6Protocol IPFamily = "IPv6" - // MaxServiceTopologyKeys is the largest number of topology keys allowed on a service - MaxServiceTopologyKeys = 16 +) + +// IPFamilyPolicyType represents the dual-stack-ness requested or required by a Service +type IPFamilyPolicyType string + +const ( + // IPFamilyPolicySingleStack indicates that this service is required to have a single IPFamily. + // The IPFamily assigned is based on the default IPFamily used by the cluster + // or as identified by service.spec.ipFamilies field + IPFamilyPolicySingleStack IPFamilyPolicyType = "SingleStack" + // IPFamilyPolicyPreferDualStack indicates that this service prefers dual-stack when + // the cluster is configured for dual-stack. If the cluster is not configured + // for dual-stack the service will be assigned a single IPFamily. If the IPFamily is not + // set in service.spec.ipFamilies then the service will be assigned the default IPFamily + // configured on the cluster + IPFamilyPolicyPreferDualStack IPFamilyPolicyType = "PreferDualStack" + // IPFamilyPolicyRequireDualStack indicates that this service requires dual-stack. Using + // IPFamilyPolicyRequireDualStack on a single stack cluster will result in validation errors. The + // IPFamilies (and their order) assigned to this service is based on service.spec.ipFamilies. If + // service.spec.ipFamilies was not provided then it will be assigned according to how they are + // configured on the cluster. If service.spec.ipFamilies has only one entry then the alternative + // IPFamily will be added by apiserver + IPFamilyPolicyRequireDualStack IPFamilyPolicyType = "RequireDualStack" ) // ServiceSpec describes the attributes that a user creates on a service @@ -3565,6 +3591,36 @@ type ServiceSpec struct { // +optional ClusterIP string + // ClusterIPs identifies all the ClusterIPs assigned to this + // service. ClusterIPs are assigned or reserved based on the values of + // service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are + // allowed in ClusterIPs. The IPFamily of each ClusterIP must match + // values provided in service.spec.ipFamilies. Clients using ClusterIPs must + // keep it in sync with ClusterIP (if provided) by having ClusterIP matching + // first element of ClusterIPs. + // +optional + ClusterIPs []string + + // IPFamilies identifies all the IPFamilies assigned for this Service. If a value + // was not provided for IPFamilies it will be defaulted based on the cluster + // configuration and the value of service.spec.ipFamilyPolicy. A maximum of two + // values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is + // conditionally mutable: it allows for adding or removing a secondary IPFamily, + // but it does not allow changing the primary IPFamily of the service. + // +optional + IPFamilies []IPFamily + + // IPFamilyPolicy represents the dual-stack-ness requested or required by this + // Service. If there is no value provided, then this Service will be considered + // SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), + // PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single + // IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies + // on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned + // to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs + // respectively. + // +optional + IPFamilyPolicy *IPFamilyPolicyType + // ExternalName is the external reference that kubedns or equivalent will // return as a CNAME record for this service. No proxying will be involved. // Must be a valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) @@ -3626,24 +3682,6 @@ type ServiceSpec struct { // +optional PublishNotReadyAddresses bool - // ipFamily specifies whether this Service has a preference for a particular IP family (e.g. - // IPv4 vs. IPv6) when the IPv6DualStack feature gate is enabled. In a dual-stack cluster, - // you can specify ipFamily when creating a ClusterIP Service to determine whether the - // controller will allocate an IPv4 or IPv6 IP for it, and you can specify ipFamily when - // creating a headless Service to determine whether it will have IPv4 or IPv6 Endpoints. In - // either case, if you do not specify an ipFamily explicitly, it will default to the - // cluster's primary IP family. - // This field is part of an alpha feature, and you should not make any assumptions about its - // semantics other than those described above. In particular, you should not assume that it - // can (or cannot) be changed after creation time; that it can only have the values "IPv4" - // and "IPv6"; or that its current value on a given Service correctly reflects the current - // state of that Service. (For ClusterIP Services, look at clusterIP to see if the Service - // is IPv4 or IPv6. For headless Services, look at the endpoints, which may be dual-stack in - // the future. For ExternalName Services, ipFamily has no meaning, but it may be set to an - // irrelevant value anyway.) - // +optional - IPFamily *IPFamily - // topologyKeys is a preference-order list of topology keys which // implementations of services should use to preferentially sort endpoints // when accessing this Service, it can not be used at the same time as diff --git a/pkg/apis/core/v1/BUILD b/pkg/apis/core/v1/BUILD index 8543d8a01a0..2e20bd610c3 100644 --- a/pkg/apis/core/v1/BUILD +++ b/pkg/apis/core/v1/BUILD @@ -27,7 +27,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/pkg/apis/core/v1/defaults.go b/pkg/apis/core/v1/defaults.go index dc8bf7e0734..560c8ea0a97 100644 --- a/pkg/apis/core/v1/defaults.go +++ b/pkg/apis/core/v1/defaults.go @@ -27,7 +27,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" - utilnet "k8s.io/utils/net" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -137,32 +136,36 @@ func SetDefaults_Service(obj *v1.Service) { obj.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster } - // if dualstack feature gate is on then we need to default - // Spec.IPFamily correctly. This is to cover the case - // when an existing cluster have been converted to dualstack - // i.e. it already contain services with Spec.IPFamily==nil - if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && - obj.Spec.Type != v1.ServiceTypeExternalName && - obj.Spec.ClusterIP != "" && /*has an ip already set*/ - obj.Spec.ClusterIP != "None" && /* not converting from ExternalName to other */ - obj.Spec.IPFamily == nil /* family was not previously set */ { + if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + // Default obj.Spec.IPFamilyPolicy if we *know* we can, otherwise it will + // be handled later in allocation. + if obj.Spec.Type != v1.ServiceTypeExternalName { + if obj.Spec.IPFamilyPolicy == nil { + if len(obj.Spec.ClusterIPs) == 2 || len(obj.Spec.IPFamilies) == 2 { + requireDualStack := v1.IPFamilyPolicyRequireDualStack + obj.Spec.IPFamilyPolicy = &requireDualStack + } + } - // there is a change that the ClusterIP (set by user) is unparsable. - // in this case, the family will be set mistakenly to ipv4 (because - // the util function does not parse errors *sigh*). The error - // will be caught in validation which asserts the validity of the - // IP and the service object will not be persisted with the wrong IP - // family + // If the user demanded dual-stack, but only specified one family, we add + // the other. + if obj.Spec.IPFamilyPolicy != nil && *(obj.Spec.IPFamilyPolicy) == v1.IPFamilyPolicyRequireDualStack && len(obj.Spec.IPFamilies) == 1 { + if obj.Spec.IPFamilies[0] == v1.IPv4Protocol { + obj.Spec.IPFamilies = append(obj.Spec.IPFamilies, v1.IPv6Protocol) + } else { + obj.Spec.IPFamilies = append(obj.Spec.IPFamilies, v1.IPv4Protocol) + } - ipv6 := v1.IPv6Protocol - ipv4 := v1.IPv4Protocol - if utilnet.IsIPv6String(obj.Spec.ClusterIP) { - obj.Spec.IPFamily = &ipv6 - } else { - obj.Spec.IPFamily = &ipv4 + // Any other dual-stack defaulting depends on cluster configuration. + // Further IPFamilies, IPFamilyPolicy defaulting is in ClusterIP alloc/reserve logic + // NOTE: strategy handles cases where ClusterIPs is used (but not ClusterIP). + } } - } + // any other defaulting depends on cluster configuration. + // further IPFamilies, IPFamilyPolicy defaulting is in ClusterIP alloc/reserve logic + // note: conversion logic handles cases where ClusterIPs is used (but not ClusterIP). + } } func SetDefaults_Pod(obj *v1.Pod) { // If limits are specified, but requests are not, default requests to limits diff --git a/pkg/apis/core/v1/defaults_test.go b/pkg/apis/core/v1/defaults_test.go index 14a990a0517..1b8e7692b62 100644 --- a/pkg/apis/core/v1/defaults_test.go +++ b/pkg/apis/core/v1/defaults_test.go @@ -29,15 +29,16 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/intstr" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" corev1 "k8s.io/kubernetes/pkg/apis/core/v1" - "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" // ensure types are installed _ "k8s.io/kubernetes/pkg/apis/core/install" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" ) // TestWorkloadDefaults detects changes to defaults within PodTemplateSpec. @@ -1023,140 +1024,6 @@ func TestSetDefaultService(t *testing.T) { } } -func TestSetDefaultServiceIPFamily(t *testing.T) { - svc := v1.Service{ - Spec: v1.ServiceSpec{ - SessionAffinity: v1.ServiceAffinityNone, - Type: v1.ServiceTypeClusterIP, - }, - } - testCases := []struct { - name string - inSvcTweak func(s v1.Service) v1.Service - outSvcTweak func(s v1.Service) v1.Service - enableDualStack bool - }{ - { - name: "dualstack off. ipfamily not set", - inSvcTweak: func(s v1.Service) v1.Service { return s }, - outSvcTweak: func(s v1.Service) v1.Service { return s }, - enableDualStack: false, - }, - { - name: "dualstack on. ipfamily not set, service is *not* ClusterIP-able", - inSvcTweak: func(s v1.Service) v1.Service { - s.Spec.Type = v1.ServiceTypeExternalName - return s - }, - outSvcTweak: func(s v1.Service) v1.Service { return s }, - enableDualStack: true, - }, - { - name: "dualstack off. ipfamily set", - inSvcTweak: func(s v1.Service) v1.Service { - ipv4Service := v1.IPv4Protocol - s.Spec.IPFamily = &ipv4Service - return s - }, - outSvcTweak: func(s v1.Service) v1.Service { - ipv4Service := v1.IPv4Protocol - s.Spec.IPFamily = &ipv4Service - return s - }, - enableDualStack: false, - }, - { - name: "dualstack off. ipfamily not set. clusterip set", - inSvcTweak: func(s v1.Service) v1.Service { - s.Spec.ClusterIP = "1.1.1.1" - return s - }, - outSvcTweak: func(s v1.Service) v1.Service { - return s - }, - enableDualStack: false, - }, - { - name: "dualstack on. ipfamily not set (clusterIP is v4)", - inSvcTweak: func(s v1.Service) v1.Service { - s.Spec.ClusterIP = "1.1.1.1" - return s - }, - outSvcTweak: func(s v1.Service) v1.Service { - ipv4Service := v1.IPv4Protocol - s.Spec.IPFamily = &ipv4Service - return s - }, - enableDualStack: true, - }, - { - name: "dualstack on. ipfamily not set (clusterIP is v6)", - inSvcTweak: func(s v1.Service) v1.Service { - s.Spec.ClusterIP = "fdd7:7713:8917:77ed:ffff:ffff:ffff:ffff" - return s - }, - outSvcTweak: func(s v1.Service) v1.Service { - ipv6Service := v1.IPv6Protocol - s.Spec.IPFamily = &ipv6Service - return s - }, - enableDualStack: true, - }, - { - name: "dualstack on. ipfamily set (clusterIP is v4)", - inSvcTweak: func(s v1.Service) v1.Service { - ipv4Service := v1.IPv4Protocol - s.Spec.IPFamily = &ipv4Service - s.Spec.ClusterIP = "1.1.1.1" - return s - }, - outSvcTweak: func(s v1.Service) v1.Service { - ipv4Service := v1.IPv4Protocol - s.Spec.IPFamily = &ipv4Service - return s - }, - enableDualStack: true, - }, - { - name: "dualstack on. ipfamily set (clusterIP is v6)", - inSvcTweak: func(s v1.Service) v1.Service { - ipv6Service := v1.IPv6Protocol - s.Spec.IPFamily = &ipv6Service - s.Spec.ClusterIP = "fdd7:7713:8917:77ed:ffff:ffff:ffff:ffff" - return s - }, - outSvcTweak: func(s v1.Service) v1.Service { - ipv6Service := v1.IPv6Protocol - s.Spec.IPFamily = &ipv6Service - return s - }, - enableDualStack: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - tweakedIn := tc.inSvcTweak(svc) - expectedSvc := tc.outSvcTweak(svc) - defaulted := roundTrip(t, runtime.Object(&tweakedIn)) - - defaultedSvc := defaulted.(*v1.Service) - if expectedSvc.Spec.IPFamily != nil { - if defaultedSvc.Spec.IPFamily == nil { - t.Fatalf("defaulted service ipfamily is nil while expected is not") - } - if *(expectedSvc.Spec.IPFamily) != *(defaultedSvc.Spec.IPFamily) { - t.Fatalf("defaulted service ipfamily %v does not match expected %v", defaultedSvc.Spec.IPFamily, expectedSvc.Spec.IPFamily) - } - } - - if expectedSvc.Spec.IPFamily == nil && defaultedSvc.Spec.IPFamily != nil { - t.Fatalf("defaulted service ipfamily is not nil, while expected service ipfamily is") - } - }) - } -} - func TestSetDefaultServiceSessionAffinityConfig(t *testing.T) { testCases := map[string]v1.Service{ "SessionAffinityConfig is empty": { @@ -1935,3 +1802,253 @@ func TestSetDefaultEnableServiceLinks(t *testing.T) { t.Errorf("Expected enableServiceLinks value: %+v\ngot: %+v\n", v1.DefaultEnableServiceLinks, *output.Spec.EnableServiceLinks) } } + +func TestSetDefaultIPFamilies(t *testing.T) { + preferDualStack := v1.IPFamilyPolicyPreferDualStack + requireDualStack := v1.IPFamilyPolicyRequireDualStack + testCases := []struct { + name string + expectedIPFamiliesWithGate []v1.IPFamily + svc v1.Service + }{ + { + name: "must not set for ExternalName", + expectedIPFamiliesWithGate: nil, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeExternalName, + }, + }, + }, + { + name: "must not set for single stack", + expectedIPFamiliesWithGate: nil, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + }, + }, + }, + { + name: "must not set for single stack, even when a family is set", + expectedIPFamiliesWithGate: []v1.IPFamily{v1.IPv6Protocol}, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, + }, + }, + { + name: "must not set for preferDualStack", + expectedIPFamiliesWithGate: []v1.IPFamily{v1.IPv6Protocol}, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, + }, + }, + { + name: "must set for requireDualStack (6,4)", + expectedIPFamiliesWithGate: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, + }, + }, + { + name: "must set for requireDualStack (4,6)", + expectedIPFamiliesWithGate: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + }, + } + for _, test := range testCases { + // run with gate + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + obj2 := roundTrip(t, runtime.Object(&test.svc)) + svc2 := obj2.(*v1.Service) + + if len(test.expectedIPFamiliesWithGate) != len(svc2.Spec.IPFamilies) { + t.Fatalf("expected .spec.IPFamilies len:%v got %v", len(test.expectedIPFamiliesWithGate), len(svc2.Spec.IPFamilies)) + } + + for i, family := range test.expectedIPFamiliesWithGate { + if svc2.Spec.IPFamilies[i] != family { + t.Fatalf("failed. expected family %v at %v got %v", family, i, svc2.Spec.IPFamilies[i]) + } + } + }) + + // run without gate (families should not change) + t.Run(fmt.Sprintf("without-gate:%s", test.name), func(t *testing.T) { + obj2 := roundTrip(t, runtime.Object(&test.svc)) + svc2 := obj2.(*v1.Service) + + if len(test.svc.Spec.IPFamilies) != len(svc2.Spec.IPFamilies) { + t.Fatalf("expected .spec.IPFamilies len:%v got %v", len(test.expectedIPFamiliesWithGate), len(svc2.Spec.IPFamilies)) + } + + for i, family := range test.svc.Spec.IPFamilies { + if svc2.Spec.IPFamilies[i] != family { + t.Fatalf("failed. expected family %v at %v got %v", family, i, svc2.Spec.IPFamilies[i]) + } + } + }) + + } +} + +func TestSetDefaultServiceIPFamilyPolicy(t *testing.T) { + singleStack := v1.IPFamilyPolicySingleStack + preferDualStack := v1.IPFamilyPolicyPreferDualStack + requireDualStack := v1.IPFamilyPolicyRequireDualStack + + testCases := []struct { + name string + expectedIPfamilyPolicy *v1.IPFamilyPolicyType + svc v1.Service + }{ + { + name: "must not set for ExternalName", + expectedIPfamilyPolicy: nil, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeExternalName, + }, + }, + }, + { + name: "must not set for ExternalName even with semantically valid data", + expectedIPfamilyPolicy: nil, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeExternalName, + ClusterIPs: []string{"1.2.3.4", "2001::1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, + }, + }, + { + name: "must set if there are more than one ip", + expectedIPfamilyPolicy: &requireDualStack, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"1.2.3.4", "2001::1"}, + }, + }, + }, + { + name: "must set if there are more than one ip family", + expectedIPfamilyPolicy: &requireDualStack, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, + }, + }, + { + name: "must not set if there is one ip", + expectedIPfamilyPolicy: nil, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"1.2.3.4"}, + }, + }, + }, + { + name: "must not set if there is one ip family", + expectedIPfamilyPolicy: nil, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + }, + { + name: "must not override user input", + expectedIPfamilyPolicy: &singleStack, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilyPolicy: &singleStack, + }, + }, + }, + { + name: "must not override user input/ preferDualStack", + expectedIPfamilyPolicy: &preferDualStack, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilyPolicy: &preferDualStack, + }, + }, + }, + + { + name: "must not override user input even when it is invalid", + expectedIPfamilyPolicy: &preferDualStack, + + svc: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + IPFamilyPolicy: &preferDualStack, + }, + }, + }, + } + + for _, test := range testCases { + // with gate + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + obj2 := roundTrip(t, runtime.Object(&test.svc)) + svc2 := obj2.(*v1.Service) + + if test.expectedIPfamilyPolicy == nil && svc2.Spec.IPFamilyPolicy != nil { + t.Fatalf("expected .spec.PreferDualStack to be unset (nil)") + } + if test.expectedIPfamilyPolicy != nil && (svc2.Spec.IPFamilyPolicy == nil || *(svc2.Spec.IPFamilyPolicy) != *(test.expectedIPfamilyPolicy)) { + t.Fatalf("expected .spec.PreferDualStack to be set to %v got %v", *(test.expectedIPfamilyPolicy), svc2.Spec.IPFamilyPolicy) + } + }) + + // without gate. IPFamilyPolicy should never change + t.Run(test.name, func(t *testing.T) { + obj2 := roundTrip(t, runtime.Object(&test.svc)) + svc2 := obj2.(*v1.Service) + + if test.svc.Spec.IPFamilyPolicy == nil && svc2.Spec.IPFamilyPolicy != nil { + t.Fatalf("expected .spec.PreferDualStack to be unset (nil)") + } + if test.svc.Spec.IPFamilyPolicy != nil && (svc2.Spec.IPFamilyPolicy == nil || *(svc2.Spec.IPFamilyPolicy) != *(test.expectedIPfamilyPolicy)) { + t.Fatalf("expected .spec.PreferDualStack to be set to %v got %v", *(test.expectedIPfamilyPolicy), svc2.Spec.IPFamilyPolicy) + } + }) + + } +} diff --git a/pkg/apis/core/v1/zz_generated.conversion.go b/pkg/apis/core/v1/zz_generated.conversion.go index 914461f7de5..e7e8951a51c 100644 --- a/pkg/apis/core/v1/zz_generated.conversion.go +++ b/pkg/apis/core/v1/zz_generated.conversion.go @@ -7582,6 +7582,7 @@ func autoConvert_v1_ServiceSpec_To_core_ServiceSpec(in *v1.ServiceSpec, out *cor out.Ports = *(*[]core.ServicePort)(unsafe.Pointer(&in.Ports)) out.Selector = *(*map[string]string)(unsafe.Pointer(&in.Selector)) out.ClusterIP = in.ClusterIP + out.ClusterIPs = *(*[]string)(unsafe.Pointer(&in.ClusterIPs)) out.Type = core.ServiceType(in.Type) out.ExternalIPs = *(*[]string)(unsafe.Pointer(&in.ExternalIPs)) out.SessionAffinity = core.ServiceAffinity(in.SessionAffinity) @@ -7592,8 +7593,9 @@ func autoConvert_v1_ServiceSpec_To_core_ServiceSpec(in *v1.ServiceSpec, out *cor out.HealthCheckNodePort = in.HealthCheckNodePort out.PublishNotReadyAddresses = in.PublishNotReadyAddresses out.SessionAffinityConfig = (*core.SessionAffinityConfig)(unsafe.Pointer(in.SessionAffinityConfig)) - out.IPFamily = (*core.IPFamily)(unsafe.Pointer(in.IPFamily)) + out.IPFamilies = *(*[]core.IPFamily)(unsafe.Pointer(&in.IPFamilies)) out.TopologyKeys = *(*[]string)(unsafe.Pointer(&in.TopologyKeys)) + out.IPFamilyPolicy = (*core.IPFamilyPolicyType)(unsafe.Pointer(in.IPFamilyPolicy)) return nil } @@ -7607,6 +7609,9 @@ func autoConvert_core_ServiceSpec_To_v1_ServiceSpec(in *core.ServiceSpec, out *v out.Ports = *(*[]v1.ServicePort)(unsafe.Pointer(&in.Ports)) out.Selector = *(*map[string]string)(unsafe.Pointer(&in.Selector)) out.ClusterIP = in.ClusterIP + out.ClusterIPs = *(*[]string)(unsafe.Pointer(&in.ClusterIPs)) + out.IPFamilies = *(*[]v1.IPFamily)(unsafe.Pointer(&in.IPFamilies)) + out.IPFamilyPolicy = (*v1.IPFamilyPolicyType)(unsafe.Pointer(in.IPFamilyPolicy)) out.ExternalName = in.ExternalName out.ExternalIPs = *(*[]string)(unsafe.Pointer(&in.ExternalIPs)) out.LoadBalancerIP = in.LoadBalancerIP @@ -7616,7 +7621,6 @@ func autoConvert_core_ServiceSpec_To_v1_ServiceSpec(in *core.ServiceSpec, out *v out.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(in.ExternalTrafficPolicy) out.HealthCheckNodePort = in.HealthCheckNodePort out.PublishNotReadyAddresses = in.PublishNotReadyAddresses - out.IPFamily = (*v1.IPFamily)(unsafe.Pointer(in.IPFamily)) out.TopologyKeys = *(*[]string)(unsafe.Pointer(&in.TopologyKeys)) return nil } diff --git a/pkg/apis/core/validation/BUILD b/pkg/apis/core/validation/BUILD index ab1f47c34b3..5765edd78cd 100644 --- a/pkg/apis/core/validation/BUILD +++ b/pkg/apis/core/validation/BUILD @@ -7,7 +7,6 @@ load( go_library( name = "go_default_library", srcs = [ - "conditional_validation.go", "doc.go", "events.go", "validation.go", @@ -49,7 +48,6 @@ go_library( go_test( name = "go_default_test", srcs = [ - "conditional_validation_test.go", "events_test.go", "validation_test.go", ], @@ -63,7 +61,6 @@ go_test( "//staging/src/k8s.io/api/events/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", diff --git a/pkg/apis/core/validation/conditional_validation.go b/pkg/apis/core/validation/conditional_validation.go deleted file mode 100644 index cc2393d4de9..00000000000 --- a/pkg/apis/core/validation/conditional_validation.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validation - -import ( - "fmt" - "net" - "strings" - - "k8s.io/apimachinery/pkg/util/validation/field" - utilfeature "k8s.io/apiserver/pkg/util/feature" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/features" - netutils "k8s.io/utils/net" -) - -// ValidateConditionalService validates conditionally valid fields. allowedIPFamilies is an ordered -// list of the valid IP families (IPv4 or IPv6) that are supported. The first family in the slice -// is the cluster default, although the clusterIP here dictates the family defaulting. -func ValidateConditionalService(service, oldService *api.Service, allowedIPFamilies []api.IPFamily) field.ErrorList { - var errs field.ErrorList - - errs = append(errs, validateIPFamily(service, oldService, allowedIPFamilies)...) - - return errs -} - -// validateIPFamily checks the IPFamily field. -func validateIPFamily(service, oldService *api.Service, allowedIPFamilies []api.IPFamily) field.ErrorList { - var errs field.ErrorList - - // specifically allow an invalid value to remain in storage as long as the user isn't changing it, regardless of gate - if oldService != nil && oldService.Spec.IPFamily != nil && service.Spec.IPFamily != nil && *oldService.Spec.IPFamily == *service.Spec.IPFamily { - return errs - } - - // If the gate is off, setting or changing IPFamily is not allowed, but clearing it is - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - if service.Spec.IPFamily != nil { - if oldService != nil { - errs = append(errs, ValidateImmutableField(service.Spec.IPFamily, oldService.Spec.IPFamily, field.NewPath("spec", "ipFamily"))...) - } else { - errs = append(errs, field.Forbidden(field.NewPath("spec", "ipFamily"), "programmer error, must be cleared when the dual-stack feature gate is off")) - } - } - return errs - } - - // PrepareCreate, PrepareUpdate, and test cases must all set IPFamily when the gate is on - if service.Spec.IPFamily == nil { - errs = append(errs, field.Required(field.NewPath("spec", "ipFamily"), "programmer error, must be set or defaulted by other fields")) - return errs - } - - // A user is not allowed to change the IPFamily field, except for ExternalName services - if oldService != nil && oldService.Spec.IPFamily != nil && service.Spec.Type != api.ServiceTypeExternalName { - errs = append(errs, ValidateImmutableField(service.Spec.IPFamily, oldService.Spec.IPFamily, field.NewPath("spec", "ipFamily"))...) - } - - // Verify the IPFamily is one of the allowed families - desiredFamily := *service.Spec.IPFamily - if hasIPFamily(allowedIPFamilies, desiredFamily) { - // the IP family is one of the allowed families, verify that it matches cluster IP - switch ip := net.ParseIP(service.Spec.ClusterIP); { - case ip == nil: - // do not need to check anything - case netutils.IsIPv6(ip) && desiredFamily != api.IPv6Protocol: - errs = append(errs, field.Invalid(field.NewPath("spec", "ipFamily"), *service.Spec.IPFamily, "does not match IPv6 cluster IP")) - case !netutils.IsIPv6(ip) && desiredFamily != api.IPv4Protocol: - errs = append(errs, field.Invalid(field.NewPath("spec", "ipFamily"), *service.Spec.IPFamily, "does not match IPv4 cluster IP")) - } - } else { - errs = append(errs, field.Invalid(field.NewPath("spec", "ipFamily"), desiredFamily, fmt.Sprintf("only the following families are allowed: %s", joinIPFamilies(allowedIPFamilies, ", ")))) - } - return errs -} - -func hasIPFamily(families []api.IPFamily, family api.IPFamily) bool { - for _, allow := range families { - if allow == family { - return true - } - } - return false -} - -func joinIPFamilies(families []api.IPFamily, separator string) string { - var b strings.Builder - for i, family := range families { - if i != 0 { - b.WriteString(separator) - } - b.WriteString(string(family)) - } - return b.String() -} diff --git a/pkg/apis/core/validation/conditional_validation_test.go b/pkg/apis/core/validation/conditional_validation_test.go deleted file mode 100644 index fe1c7737ab3..00000000000 --- a/pkg/apis/core/validation/conditional_validation_test.go +++ /dev/null @@ -1,266 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validation - -import ( - "reflect" - "strings" - "testing" - - "k8s.io/apimachinery/pkg/util/diff" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/features" -) - -func TestValidateServiceIPFamily(t *testing.T) { - ipv4 := api.IPv4Protocol - ipv6 := api.IPv6Protocol - var unknown api.IPFamily = "Unknown" - testCases := []struct { - name string - dualStackEnabled bool - ipFamilies []api.IPFamily - svc *api.Service - oldSvc *api.Service - expectErr []string - }{ - { - name: "allowed ipv4", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv4, - }, - }, - }, - { - name: "allowed ipv6", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv6Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv6, - }, - }, - }, - { - name: "allowed ipv4 dual stack default IPv6", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv4, - }, - }, - }, - { - name: "allowed ipv4 dual stack default IPv4", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv4, - }, - }, - }, - { - name: "allowed ipv6 dual stack default IPv6", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv6, - }, - }, - }, - { - name: "allowed ipv6 dual stack default IPv4", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv6, - }, - }, - }, - { - name: "allow ipfamily to remain invalid if update doesn't change it", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &unknown, - }, - }, - oldSvc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &unknown, - }, - }, - }, - { - name: "not allowed ipfamily/clusterip mismatch", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv4, - ClusterIP: "ffd0::1", - }, - }, - expectErr: []string{"spec.ipFamily: Invalid value: \"IPv4\": does not match IPv6 cluster IP"}, - }, - { - name: "not allowed unknown family", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &unknown, - }, - }, - expectErr: []string{"spec.ipFamily: Invalid value: \"Unknown\": only the following families are allowed: IPv4"}, - }, - { - name: "not allowed ipv4 cluster ip without family", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv6Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - ClusterIP: "127.0.0.1", - }, - }, - expectErr: []string{"spec.ipFamily: Required value: programmer error, must be set or defaulted by other fields"}, - }, - { - name: "not allowed ipv6 cluster ip without family", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - ClusterIP: "ffd0::1", - }, - }, - expectErr: []string{"spec.ipFamily: Required value: programmer error, must be set or defaulted by other fields"}, - }, - - { - name: "not allowed to change ipfamily for default type", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv4, - }, - }, - oldSvc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv6, - }, - }, - expectErr: []string{"spec.ipFamily: Invalid value: \"IPv4\": field is immutable"}, - }, - { - name: "allowed to change ipfamily for external name", - dualStackEnabled: true, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeExternalName, - IPFamily: &ipv4, - }, - }, - oldSvc: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeExternalName, - IPFamily: &ipv6, - }, - }, - }, - - { - name: "ipfamily allowed to be empty when dual stack is off", - dualStackEnabled: false, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - ClusterIP: "127.0.0.1", - }, - }, - }, - { - name: "ipfamily must be empty when dual stack is off", - dualStackEnabled: false, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - IPFamily: &ipv4, - ClusterIP: "127.0.0.1", - }, - }, - expectErr: []string{"spec.ipFamily: Forbidden: programmer error, must be cleared when the dual-stack feature gate is off"}, - }, - { - name: "ipfamily allowed to be cleared when dual stack is off", - dualStackEnabled: false, - ipFamilies: []api.IPFamily{api.IPv4Protocol}, - svc: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - ClusterIP: "127.0.0.1", - }, - }, - oldSvc: &api.Service{ - Spec: api.ServiceSpec{ - Type: api.ServiceTypeClusterIP, - ClusterIP: "127.0.0.1", - IPFamily: &ipv4, - }, - }, - expectErr: nil, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.dualStackEnabled)() - oldSvc := tc.oldSvc.DeepCopy() - newSvc := tc.svc.DeepCopy() - originalNewSvc := newSvc.DeepCopy() - errs := ValidateConditionalService(newSvc, oldSvc, tc.ipFamilies) - // objects should never be changed - if !reflect.DeepEqual(oldSvc, tc.oldSvc) { - t.Errorf("old object changed: %v", diff.ObjectReflectDiff(oldSvc, tc.svc)) - } - if !reflect.DeepEqual(newSvc, originalNewSvc) { - t.Errorf("new object changed: %v", diff.ObjectReflectDiff(newSvc, originalNewSvc)) - } - - if len(errs) != len(tc.expectErr) { - t.Fatalf("unexpected number of errors: %v", errs) - } - for i := range errs { - if !strings.Contains(errs[i].Error(), tc.expectErr[i]) { - t.Errorf("unexpected error %d: %v", i, errs[i]) - } - } - }) - } -} diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index 2bce665f19c..915fe7d7640 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -4139,13 +4139,16 @@ var supportedSessionAffinityType = sets.NewString(string(core.ServiceAffinityCli var supportedServiceType = sets.NewString(string(core.ServiceTypeClusterIP), string(core.ServiceTypeNodePort), string(core.ServiceTypeLoadBalancer), string(core.ServiceTypeExternalName)) +var supportedServiceIPFamily = sets.NewString(string(core.IPv4Protocol), string(core.IPv6Protocol)) +var supportedServiceIPFamilyPolicy = sets.NewString(string(core.IPFamilyPolicySingleStack), string(core.IPFamilyPolicyPreferDualStack), string(core.IPFamilyPolicyRequireDualStack)) + // ValidateService tests if required fields/annotations of a Service are valid. func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorList { allErrs := ValidateObjectMeta(&service.ObjectMeta, true, ValidateServiceName, field.NewPath("metadata")) specPath := field.NewPath("spec") - isHeadlessService := service.Spec.ClusterIP == core.ClusterIPNone - if len(service.Spec.Ports) == 0 && !isHeadlessService && service.Spec.Type != core.ServiceTypeExternalName { + + if len(service.Spec.Ports) == 0 && !isHeadlessService(service) && service.Spec.Type != core.ServiceTypeExternalName { allErrs = append(allErrs, field.Required(specPath.Child("ports"), "")) } switch service.Spec.Type { @@ -4160,16 +4163,25 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi allErrs = append(allErrs, field.Invalid(portPath, port.Port, fmt.Sprintf("may not expose port %v externally since it is used by kubelet", ports.KubeletPort))) } } - if service.Spec.ClusterIP == "None" { - allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIP"), service.Spec.ClusterIP, "may not be set to 'None' for LoadBalancer services")) + if isHeadlessService(service) { + allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIPs").Index(0), service.Spec.ClusterIPs[0], "may not be set to 'None' for LoadBalancer services")) } case core.ServiceTypeNodePort: - if service.Spec.ClusterIP == "None" { - allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIP"), service.Spec.ClusterIP, "may not be set to 'None' for NodePort services")) + if isHeadlessService(service) { + allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIPs").Index(0), service.Spec.ClusterIPs[0], "may not be set to 'None' for NodePort services")) } case core.ServiceTypeExternalName: - if service.Spec.ClusterIP != "" { - allErrs = append(allErrs, field.Forbidden(specPath.Child("clusterIP"), "must be empty for ExternalName services")) + // must have len(.spec.ClusterIPs) == 0 // note: strategy sets ClusterIPs based on ClusterIP + if len(service.Spec.ClusterIPs) > 0 { + allErrs = append(allErrs, field.Forbidden(specPath.Child("clusterIPs"), "may not be set for ExternalName services")) + } + + // must have nil families and nil policy + if len(service.Spec.IPFamilies) > 0 { + allErrs = append(allErrs, field.Forbidden(specPath.Child("ipFamilies"), "may not be set for ExternalName services")) + } + if service.Spec.IPFamilyPolicy != nil { + allErrs = append(allErrs, field.Forbidden(specPath.Child("ipFamilyPolicy"), "may not be set for ExternalName services")) } // The value (a CNAME) may have a trailing dot to denote it as fully qualified @@ -4185,7 +4197,7 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi portsPath := specPath.Child("ports") for i := range service.Spec.Ports { portPath := portsPath.Index(i) - allErrs = append(allErrs, validateServicePort(&service.Spec.Ports[i], len(service.Spec.Ports) > 1, isHeadlessService, allowAppProtocol, &allPortNames, portPath)...) + allErrs = append(allErrs, validateServicePort(&service.Spec.Ports[i], len(service.Spec.Ports) > 1, isHeadlessService(service), allowAppProtocol, &allPortNames, portPath)...) } if service.Spec.Selector != nil { @@ -4206,11 +4218,8 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi } } - if helper.IsServiceIPSet(service) { - if ip := net.ParseIP(service.Spec.ClusterIP); ip == nil { - allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIP"), service.Spec.ClusterIP, "must be empty, 'None', or a valid IP address")) - } - } + // dualstack <-> ClusterIPs <-> ipfamilies + allErrs = append(allErrs, validateServiceClusterIPsRelatedFields(service)...) ipPath := specPath.Child("externalIPs") for i, ip := range service.Spec.ExternalIPs { @@ -4338,6 +4347,7 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi } } + // external traffic fields allErrs = append(allErrs, validateServiceExternalTrafficFieldsValue(service)...) return allErrs } @@ -4446,13 +4456,16 @@ func ValidateServiceCreate(service *core.Service) field.ErrorList { func ValidateServiceUpdate(service, oldService *core.Service) field.ErrorList { allErrs := ValidateObjectMetaUpdate(&service.ObjectMeta, &oldService.ObjectMeta, field.NewPath("metadata")) - // ClusterIP should be immutable for services using it (every type other than ExternalName) - // which do not have ClusterIP assigned yet (empty string value) - if service.Spec.Type != core.ServiceTypeExternalName { - if oldService.Spec.Type != core.ServiceTypeExternalName && oldService.Spec.ClusterIP != "" { - allErrs = append(allErrs, ValidateImmutableField(service.Spec.ClusterIP, oldService.Spec.ClusterIP, field.NewPath("spec", "clusterIP"))...) - } - } + // User can upgrade (add another clusterIP or ipFamily) + // can downgrade (remove secondary clusterIP or ipFamily) + // but *CAN NOT* change primary/secondary clusterIP || ipFamily *UNLESS* + // they are changing from/to/ON ExternalName + + upgradeDowngradeClusterIPsErrs := validateUpgradeDowngradeClusterIPs(oldService, service) + allErrs = append(allErrs, upgradeDowngradeClusterIPsErrs...) + + upgradeDowngradeIPFamiliesErrs := validateUpgradeDowngradeIPFamilies(oldService, service) + allErrs = append(allErrs, upgradeDowngradeIPFamiliesErrs...) // allow AppProtocol value if the feature gate is set or the field is // already set on the resource. @@ -6094,3 +6107,255 @@ func ValidateSpreadConstraintNotRepeat(fldPath *field.Path, constraint core.Topo } return nil } + +// validateServiceClusterIPsRelatedFields validates .spec.ClusterIPs,, .spec.IPFamilies, .spec.ipFamilyPolicy +func validateServiceClusterIPsRelatedFields(service *core.Service) field.ErrorList { + // ClusterIP, ClusterIPs, IPFamilyPolicy and IPFamilies are validated prior (all must be unset) for ExternalName service + if service.Spec.Type == core.ServiceTypeExternalName { + return field.ErrorList{} + } + + allErrs := field.ErrorList{} + hasInvalidIPs := false + + specPath := field.NewPath("spec") + clusterIPsField := specPath.Child("clusterIPs") + ipFamiliesField := specPath.Child("ipFamilies") + ipFamilyPolicyField := specPath.Child("ipFamilyPolicy") + + // Make sure ClusterIP and ClusterIPs are synced. For most cases users can + // just manage one or the other and we'll handle the rest (see PrepareFor* + // in strategy). + if len(service.Spec.ClusterIP) != 0 { + // If ClusterIP is set, ClusterIPs[0] must match. + if len(service.Spec.ClusterIPs) == 0 { + allErrs = append(allErrs, field.Required(clusterIPsField, "")) + } else if service.Spec.ClusterIPs[0] != service.Spec.ClusterIP { + allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "element [0] must match clusterIP")) + } + } else { // ClusterIP == "" + // If ClusterIP is not set, ClusterIPs must also be unset. + if len(service.Spec.ClusterIPs) != 0 { + allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "must be empty when clusterIP is empty")) + } + } + + // ipfamilies stand alone validation + // must be either IPv4 or IPv6 + seen := sets.String{} + for i, ipFamily := range service.Spec.IPFamilies { + if !supportedServiceIPFamily.Has(string(ipFamily)) { + allErrs = append(allErrs, field.NotSupported(ipFamiliesField.Index(i), ipFamily, supportedServiceIPFamily.List())) + } + // no duplicate check also ensures that ipfamilies is dualstacked, in any order + if seen.Has(string(ipFamily)) { + allErrs = append(allErrs, field.Duplicate(ipFamiliesField.Index(i), ipFamily)) + } + seen.Insert(string(ipFamily)) + } + + // IPFamilyPolicy stand alone validation + //note: nil is ok, defaulted in alloc check registry/core/service/* + if service.Spec.IPFamilyPolicy != nil { + // must have a supported value + if !supportedServiceIPFamilyPolicy.Has(string(*(service.Spec.IPFamilyPolicy))) { + allErrs = append(allErrs, field.NotSupported(ipFamilyPolicyField, service.Spec.IPFamilyPolicy, supportedServiceIPFamilyPolicy.List())) + } + } + + // clusterIPs stand alone validation + // valid ips with None and empty string handling + // duplication check is done as part of DualStackvalidation below + for i, clusterIP := range service.Spec.ClusterIPs { + // valid at first location only. if and only if len(clusterIPs) == 1 + if i == 0 && clusterIP == core.ClusterIPNone { + if len(service.Spec.ClusterIPs) > 1 { + hasInvalidIPs = true + allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "'None' must be the first and only value")) + } + continue + } + + // is it valid ip? + errorMessages := validation.IsValidIP(clusterIP) + hasInvalidIPs = (len(errorMessages) != 0) || hasInvalidIPs + for _, msg := range errorMessages { + allErrs = append(allErrs, field.Invalid(clusterIPsField.Index(i), clusterIP, msg)) + } + } + + // max two + if len(service.Spec.ClusterIPs) > 2 { + allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "may only hold up to 2 values")) + } + + // at this stage if there is an invalid ip or misplaced none/empty string + // it will skew the error messages (bad index || dualstackness of already bad ips). so we + // stop here if there are errors in clusterIPs validation + if hasInvalidIPs { + return allErrs + } + + // must be dual stacked ips if they are more than one ip + if len(service.Spec.ClusterIPs) > 1 /* meaning: it does not have a None or empty string */ { + dualStack, err := netutils.IsDualStackIPStrings(service.Spec.ClusterIPs) + if err != nil { // though we check for that earlier. safe > sorry + allErrs = append(allErrs, field.InternalError(clusterIPsField, fmt.Errorf("failed to check for dual stack with error:%v", err))) + } + + // We only support one from each IP family (i.e. max two IPs in this list). + if !dualStack { + allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "may specify no more than one IP for each IP family")) + } + } + + // match clusterIPs to their families, if they were provided + if !isHeadlessService(service) && len(service.Spec.ClusterIPs) > 0 && len(service.Spec.IPFamilies) > 0 { + for i, ip := range service.Spec.ClusterIPs { + if i > (len(service.Spec.IPFamilies) - 1) { + break // no more families to check + } + + // 4=>6 + if service.Spec.IPFamilies[i] == core.IPv4Protocol && netutils.IsIPv6String(ip) { + allErrs = append(allErrs, field.Invalid(clusterIPsField.Index(i), ip, fmt.Sprintf("expected an IPv4 value as indicated by `ipFamilies[%v]`", i))) + } + // 6=>4 + if service.Spec.IPFamilies[i] == core.IPv6Protocol && !netutils.IsIPv6String(ip) { + allErrs = append(allErrs, field.Invalid(clusterIPsField.Index(i), ip, fmt.Sprintf("expected an IPv6 value as indicated by `ipFamilies[%v]`", i))) + } + } + } + + return allErrs +} + +// specific validation for clusterIPs in cases of user upgrading or downgrading to/from dualstack +func validateUpgradeDowngradeClusterIPs(oldService, service *core.Service) field.ErrorList { + allErrs := make(field.ErrorList, 0) + + // bail out early for ExternalName + if service.Spec.Type == core.ServiceTypeExternalName || oldService.Spec.Type == core.ServiceTypeExternalName { + return allErrs + } + newIsHeadless := isHeadlessService(service) + oldIsHeadless := isHeadlessService(oldService) + + if oldIsHeadless && newIsHeadless { + return allErrs + } + + switch { + // no change in ClusterIP lengths + // compare each + case len(oldService.Spec.ClusterIPs) == len(service.Spec.ClusterIPs): + for i, ip := range oldService.Spec.ClusterIPs { + if ip != service.Spec.ClusterIPs[i] { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not change once set")) + } + } + + // something has been released (downgraded) + case len(oldService.Spec.ClusterIPs) > len(service.Spec.ClusterIPs): + // primary ClusterIP has been released + if len(service.Spec.ClusterIPs) == 0 { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "primary clusterIP can not be unset")) + } + + // test if primary clusterIP has changed + if len(oldService.Spec.ClusterIPs) > 0 && + len(service.Spec.ClusterIPs) > 0 && + service.Spec.ClusterIPs[0] != oldService.Spec.ClusterIPs[0] { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "may not change once set")) + } + + // test if secondary ClusterIP has been released. has this service been downgraded correctly? + // user *must* set IPFamilyPolicy == SingleStack + if len(service.Spec.ClusterIPs) == 1 { + if service.Spec.IPFamilyPolicy == nil || *(service.Spec.IPFamilyPolicy) != core.IPFamilyPolicySingleStack { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "`ipFamilyPolicy` must be set to 'SingleStack' when releasing the secondary clusterIP")) + } + } + case len(oldService.Spec.ClusterIPs) < len(service.Spec.ClusterIPs): + // something has been added (upgraded) + // test if primary clusterIP has changed + if len(oldService.Spec.ClusterIPs) > 0 && + service.Spec.ClusterIPs[0] != oldService.Spec.ClusterIPs[0] { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "may not change once set")) + } + // we don't check for Policy == RequireDualStack here since, Validation/Creation func takes care of it + } + return allErrs +} + +// specific validation for ipFamilies in cases of user upgrading or downgrading to/from dualstack +func validateUpgradeDowngradeIPFamilies(oldService, service *core.Service) field.ErrorList { + allErrs := make(field.ErrorList, 0) + // bail out early for ExternalName + if service.Spec.Type == core.ServiceTypeExternalName || oldService.Spec.Type == core.ServiceTypeExternalName { + return allErrs + } + + oldIsHeadless := isHeadlessService(oldService) + newIsHeadless := isHeadlessService(service) + + // if changed to/from headless, then bail out + if newIsHeadless != oldIsHeadless { + return allErrs + } + // headless can change families + if newIsHeadless { + return allErrs + } + + switch { + case len(oldService.Spec.IPFamilies) == len(service.Spec.IPFamilies): + // no change in ClusterIP lengths + // compare each + + for i, ip := range oldService.Spec.IPFamilies { + if ip != service.Spec.IPFamilies[i] { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.IPFamilies, "may not change once set")) + } + } + + case len(oldService.Spec.IPFamilies) > len(service.Spec.IPFamilies): + // something has been released (downgraded) + + // test if primary ipfamily has been released + if len(service.Spec.ClusterIPs) == 0 { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.IPFamilies, "primary ipFamily can not be unset")) + } + + // test if primary ipFamily has changed + if len(service.Spec.IPFamilies) > 0 && + service.Spec.IPFamilies[0] != oldService.Spec.IPFamilies[0] { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.ClusterIPs, "may not change once set")) + } + + // test if secondary IPFamily has been released. has this service been downgraded correctly? + // user *must* set IPFamilyPolicy == SingleStack + if len(service.Spec.IPFamilies) == 1 { + if service.Spec.IPFamilyPolicy == nil || *(service.Spec.IPFamilyPolicy) != core.IPFamilyPolicySingleStack { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "`ipFamilyPolicy` must be set to 'SingleStack' when releasing the secondary ipFamily")) + } + } + case len(oldService.Spec.IPFamilies) < len(service.Spec.IPFamilies): + // something has been added (upgraded) + + // test if primary ipFamily has changed + if len(oldService.Spec.IPFamilies) > 0 && + len(service.Spec.IPFamilies) > 0 && + service.Spec.IPFamilies[0] != oldService.Spec.IPFamilies[0] { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.ClusterIPs, "may not change once set")) + } + // we don't check for Policy == RequireDualStack here since, Validation/Creation func takes care of it + } + return allErrs +} + +func isHeadlessService(service *core.Service) bool { + return service != nil && + len(service.Spec.ClusterIPs) == 1 && + service.Spec.ClusterIPs[0] == core.ClusterIPNone +} diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index e4019568918..20276c41a22 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -9609,7 +9609,6 @@ func TestValidatePodStatusUpdate(t *testing.T) { } func makeValidService() core.Service { - serviceIPFamily := core.IPv4Protocol return core.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "valid", @@ -9623,7 +9622,6 @@ func makeValidService() core.Service { SessionAffinity: "None", Type: core.ServiceTypeClusterIP, Ports: []core.ServicePort{{Name: "p", Protocol: "TCP", Port: 8675, TargetPort: intstr.FromInt(8675)}}, - IPFamily: &serviceIPFamily, }, } } @@ -9895,6 +9893,10 @@ func TestValidatePodEphemeralContainersUpdate(t *testing.T) { func TestValidateServiceCreate(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceTopology, true)() + requireDualStack := core.IPFamilyPolicyRequireDualStack + singleStack := core.IPFamilyPolicySingleStack + preferDualStack := core.IPFamilyPolicyPreferDualStack + testCases := []struct { name string tweakSvc func(svc *core.Service) // given a basic valid service, each test case can customize it @@ -10004,6 +10006,7 @@ func TestValidateServiceCreate(t *testing.T) { tweakSvc: func(s *core.Service) { s.Spec.Ports = nil s.Spec.ClusterIP = core.ClusterIPNone + s.Spec.ClusterIPs = []string{core.ClusterIPNone} }, numErrs: 0, }, @@ -10054,6 +10057,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "invalid cluster ip", tweakSvc: func(s *core.Service) { s.Spec.ClusterIP = "invalid" + s.Spec.ClusterIPs = []string{"invalid"} }, numErrs: 1, }, @@ -10084,6 +10088,7 @@ func TestValidateServiceCreate(t *testing.T) { s.Spec.Ports[0].Port = 11722 s.Spec.Ports[0].TargetPort = intstr.FromInt(11722) s.Spec.ClusterIP = core.ClusterIPNone + s.Spec.ClusterIPs = []string{core.ClusterIPNone} }, numErrs: 0, }, @@ -10093,6 +10098,7 @@ func TestValidateServiceCreate(t *testing.T) { s.Spec.Ports[0].Port = 11722 s.Spec.Ports[0].TargetPort = intstr.FromInt(11721) s.Spec.ClusterIP = core.ClusterIPNone + s.Spec.ClusterIPs = []string{core.ClusterIPNone} }, // in the v1 API, targetPorts on headless services were tolerated. // once we have version-specific validation, we can reject this on newer API versions, but until then, we have to tolerate it for compatibility. @@ -10105,6 +10111,7 @@ func TestValidateServiceCreate(t *testing.T) { s.Spec.Ports[0].Port = 11722 s.Spec.Ports[0].TargetPort = intstr.FromString("target") s.Spec.ClusterIP = core.ClusterIPNone + s.Spec.ClusterIPs = []string{core.ClusterIPNone} }, // in the v1 API, targetPorts on headless services were tolerated. // once we have version-specific validation, we can reject this on newer API versions, but until then, we have to tolerate it for compatibility. @@ -10196,20 +10203,21 @@ func TestValidateServiceCreate(t *testing.T) { { name: "valid cluster ip - none ", tweakSvc: func(s *core.Service) { - s.Spec.ClusterIP = "None" + s.Spec.ClusterIP = core.ClusterIPNone + s.Spec.ClusterIPs = []string{core.ClusterIPNone} }, numErrs: 0, }, { name: "valid cluster ip - empty", tweakSvc: func(s *core.Service) { - s.Spec.ClusterIP = "" + s.Spec.ClusterIPs = nil s.Spec.Ports[0].TargetPort = intstr.FromString("http") }, numErrs: 0, }, { - name: "valid type - cluster", + name: "valid type - clusterIP", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeClusterIP }, @@ -10438,7 +10446,6 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid ExternalName", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeExternalName - s.Spec.ClusterIP = "" s.Spec.ExternalName = "foo.bar.example.com" }, numErrs: 0, @@ -10447,7 +10454,6 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid ExternalName (trailing dot)", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeExternalName - s.Spec.ClusterIP = "" s.Spec.ExternalName = "foo.bar.example.com." }, numErrs: 0, @@ -10457,6 +10463,7 @@ func TestValidateServiceCreate(t *testing.T) { tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeExternalName s.Spec.ClusterIP = "1.2.3.4" + s.Spec.ClusterIPs = []string{"1.2.3.4"} s.Spec.ExternalName = "foo.bar.example.com" }, numErrs: 1, @@ -10466,6 +10473,7 @@ func TestValidateServiceCreate(t *testing.T) { tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeExternalName s.Spec.ClusterIP = "None" + s.Spec.ClusterIPs = []string{"None"} s.Spec.ExternalName = "foo.bar.example.com" }, numErrs: 1, @@ -10474,7 +10482,6 @@ func TestValidateServiceCreate(t *testing.T) { name: "invalid ExternalName (not a DNS name)", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeExternalName - s.Spec.ClusterIP = "" s.Spec.ExternalName = "-123" }, numErrs: 1, @@ -10483,6 +10490,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "LoadBalancer type cannot have None ClusterIP", tweakSvc: func(s *core.Service) { s.Spec.ClusterIP = "None" + s.Spec.ClusterIPs = []string{"None"} s.Spec.Type = core.ServiceTypeLoadBalancer }, numErrs: 1, @@ -10493,6 +10501,7 @@ func TestValidateServiceCreate(t *testing.T) { s.Spec.Type = core.ServiceTypeNodePort s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)}) s.Spec.ClusterIP = "None" + s.Spec.ClusterIPs = []string{"None"} }, numErrs: 1, }, @@ -10550,29 +10559,385 @@ func TestValidateServiceCreate(t *testing.T) { }, numErrs: 1, }, + /* ip families validation */ { - name: "valid, nil service IPFamily", - tweakSvc: func(s *core.Service) { - s.Spec.IPFamily = nil - }, - numErrs: 0, - }, - { - name: "valid, service with valid IPFamily", - tweakSvc: func(s *core.Service) { - ipv4Service := core.IPv4Protocol - s.Spec.IPFamily = &ipv4Service - }, - numErrs: 0, - }, - { - name: "allowed valid, service with invalid IPFamily is ignored (tested in conditional validation)", + name: "invalid, service with invalid ipFamilies", tweakSvc: func(s *core.Service) { invalidServiceIPFamily := core.IPFamily("not-a-valid-ip-family") - s.Spec.IPFamily = &invalidServiceIPFamily + s.Spec.IPFamilies = []core.IPFamily{invalidServiceIPFamily} + }, + numErrs: 1, + }, + { + name: "invalid, service with invalid ipFamilies (2nd)", + tweakSvc: func(s *core.Service) { + invalidServiceIPFamily := core.IPFamily("not-a-valid-ip-family") + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, invalidServiceIPFamily} + }, + numErrs: 1, + }, + { + name: "IPFamilyPolicy(singleStack) is set for two families", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &singleStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, // this validated in alloc code. + }, + { + name: "valid, IPFamilyPolicy(preferDualStack) is set for two families (note: alloc sets families)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &preferDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} }, numErrs: 0, }, + + { + name: "invalid, service with 2+ ipFamilies", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol, core.IPv4Protocol} + }, + numErrs: 1, + }, + { + name: "invalid, service with same ip families", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol, core.IPv6Protocol} + }, + numErrs: 1, + }, + { + name: "valid, nil service ipFamilies", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilies = nil + }, + numErrs: 0, + }, + { + name: "valid, service with valid ipFamilies (v4)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "valid, service with valid ipFamilies (v6)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid, service with valid ipFamilies(v4,v6)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid, service with valid ipFamilies(v6,v4)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol, core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "valid, service preferred dual stack with single family", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &preferDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + }, + numErrs: 0, + }, + /* cluster IPs. some tests are reduntant */ + { + name: "invalid, garbage single ip", + tweakSvc: func(s *core.Service) { + s.Spec.ClusterIP = "garbage-ip" + s.Spec.ClusterIPs = []string{"garbage-ip"} + }, + numErrs: 1, + }, + { + name: "invalid, garbage ips", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "garbage-ip" + s.Spec.ClusterIPs = []string{"garbage-ip", "garbage-second-ip"} + }, + numErrs: 2, + }, + { + name: "invalid, garbage first ip", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "garbage-ip" + s.Spec.ClusterIPs = []string{"garbage-ip", "2001::1"} + }, + numErrs: 1, + }, + { + name: "invalid, garbage second ip", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1", "garbage-ip"} + }, + numErrs: 1, + }, + { + name: "invalid, NONE + IP", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "None" + s.Spec.ClusterIPs = []string{"None", "2001::1"} + }, + numErrs: 1, + }, + { + name: "invalid, IP + NONE", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1", "None"} + }, + numErrs: 1, + }, + { + name: "invalid, EMPTY STRING + IP", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "" + s.Spec.ClusterIPs = []string{"", "2001::1"} + }, + numErrs: 2, + }, + { + name: "invalid, IP + EMPTY STRING", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1", ""} + }, + numErrs: 1, + }, + { + name: "invalid, same ip family (v6)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1", "2001::4"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 2, + }, + { + name: "invalid, same ip family (v4)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1", "10.0.0.10"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + }, + numErrs: 2, + }, + { + name: "invalid, more than two ips", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1", "2001::1", "10.0.0.10"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 1, + }, + { + name: " multi ip, dualstack not set (request for downgrade)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &singleStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1", "2001::1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid, headless-no-selector + multi family + gate off", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "None" + s.Spec.ClusterIPs = []string{"None"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + s.Spec.Selector = nil + }, + numErrs: 0, + }, + { + name: "valid, multi ip, single ipfamilies preferDualStack", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &preferDualStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1", "2001::1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + + { + name: "valid, multi ip, single ipfamilies (must match when provided) + requireDualStack", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1", "2001::1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "invalid, families don't match (v4=>v6)", + tweakSvc: func(s *core.Service) { + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + }, + numErrs: 1, + }, + { + name: "invalid, families don't match (v6=>v4)", + tweakSvc: func(s *core.Service) { + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 1, + }, + { + name: "valid. no field set", + tweakSvc: func(s *core.Service) { + }, + numErrs: 0, + }, + + { + name: "valid, single ip", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &singleStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1"} + }, + numErrs: 0, + }, + { + name: "valid, single family", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &singleStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + + }, + numErrs: 0, + }, + { + name: "valid, single ip + single family", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &singleStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + + }, + numErrs: 0, + }, + { + name: "valid, single ip + single family (dual stack requested)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &preferDualStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + + }, + numErrs: 0, + }, + { + name: "valid, single ip, multi ipfamilies", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid, multi ips, multi ipfamilies (4,6)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1", "2001::1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid, ips, multi ipfamilies (6,4)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1", "10.0.0.1"} + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol, core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "valid, multi ips (6,4)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "2001::1" + s.Spec.ClusterIPs = []string{"2001::1", "10.0.0.1"} + }, + numErrs: 0, + }, + { + name: "valid, multi ipfamilies (6,4)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol, core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "valid, multi ips (4,6)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "10.0.0.1" + s.Spec.ClusterIPs = []string{"10.0.0.1", "2001::1"} + }, + numErrs: 0, + }, + { + name: "valid, multi ipfamilies (4,6)", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid, dual stack", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + }, + numErrs: 0, + }, + + /* toplogy keys */ { name: "valid topology keys", tweakSvc: func(s *core.Service) { @@ -10672,16 +11037,35 @@ func TestValidateServiceCreate(t *testing.T) { appProtocolEnabled: true, numErrs: 1, }, + + { + name: "invalid cluster ip != clusterIP in multi ip service", + tweakSvc: func(s *core.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIP = "10.0.0.10" + s.Spec.ClusterIPs = []string{"10.0.0.1", "2001::1"} + }, + numErrs: 1, + }, + { + name: "invalid cluster ip != clusterIP in single ip service", + tweakSvc: func(s *core.Service) { + s.Spec.ClusterIP = "10.0.0.10" + s.Spec.ClusterIPs = []string{"10.0.0.1"} + }, + numErrs: 1, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceAppProtocol, true)() + svc := makeValidService() tc.tweakSvc(&svc) errs := ValidateServiceCreate(&svc) if len(errs) != tc.numErrs { - t.Errorf("Unexpected error list for case %q: %v", tc.name, errs.ToAggregate()) + t.Errorf("Unexpected error list for case %q(expected:%v got %v) - Errors:\n %v", tc.name, tc.numErrs, len(errs), errs.ToAggregate()) } }) } @@ -12152,6 +12536,9 @@ func TestValidateNodeUpdate(t *testing.T) { } func TestValidateServiceUpdate(t *testing.T) { + requireDualStack := core.IPFamilyPolicyRequireDualStack + preferDualStack := core.IPFamilyPolicyPreferDualStack + singleStack := core.IPFamilyPolicySingleStack testCases := []struct { name string tweakSvc func(oldSvc, newSvc *core.Service) // given basic valid services, each test case can customize them @@ -12197,7 +12584,10 @@ func TestValidateServiceUpdate(t *testing.T) { name: "change cluster IP", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "8.6.7.5" + newSvc.Spec.ClusterIPs = []string{"8.6.7.5"} }, numErrs: 1, }, @@ -12205,7 +12595,10 @@ func TestValidateServiceUpdate(t *testing.T) { name: "remove cluster IP", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "" + newSvc.Spec.ClusterIPs = nil }, numErrs: 1, }, @@ -12272,26 +12665,48 @@ func TestValidateServiceUpdate(t *testing.T) { name: "LoadBalancer type cannot have None ClusterIP", tweakSvc: func(oldSvc, newSvc *core.Service) { newSvc.Spec.ClusterIP = "None" + newSvc.Spec.ClusterIPs = []string{"None"} newSvc.Spec.Type = core.ServiceTypeLoadBalancer }, numErrs: 1, }, { - name: "`None` ClusterIP cannot be changed", + name: "`None` ClusterIP can NOT be changed", tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.ClusterIP = "None" + oldSvc.Spec.ClusterIPs = []string{"None"} + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} }, numErrs: 1, }, { - name: "`None` ClusterIP cannot be removed", + name: "`None` ClusterIP can NOT be removed", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.ClusterIP = "None" + oldSvc.Spec.ClusterIPs = []string{"None"} + newSvc.Spec.ClusterIP = "" + newSvc.Spec.ClusterIPs = nil }, numErrs: 1, }, + { + name: "ClusterIP can NOT be changed to None", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + + newSvc.Spec.ClusterIP = "None" + newSvc.Spec.ClusterIPs = []string{"None"} + }, + numErrs: 1, + }, + { name: "Service with ClusterIP type cannot change its set ClusterIP", tweakSvc: func(oldSvc, newSvc *core.Service) { @@ -12299,7 +12714,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12310,7 +12728,9 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12321,7 +12741,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12332,7 +12755,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12343,7 +12769,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeLoadBalancer oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12354,7 +12783,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeLoadBalancer oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12365,7 +12797,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12376,7 +12811,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12387,7 +12825,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12398,7 +12839,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12409,7 +12853,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeLoadBalancer oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12420,7 +12867,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeLoadBalancer oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12431,7 +12881,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeLoadBalancer oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12442,7 +12895,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeLoadBalancer oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12453,7 +12909,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12464,7 +12923,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12475,7 +12937,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 1, }, @@ -12486,7 +12951,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12497,7 +12965,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12508,7 +12979,10 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} }, numErrs: 0, }, @@ -12522,84 +12996,423 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.Ports = append(newSvc.Spec.Ports, core.ServicePort{Name: "q", Port: 1, Protocol: "TCP", NodePort: 1, TargetPort: intstr.FromInt(1)}) oldSvc.Spec.ClusterIP = "" + oldSvc.Spec.ClusterIPs = nil + newSvc.Spec.ClusterIP = "None" + newSvc.Spec.ClusterIPs = []string{"None"} }, numErrs: 1, }, /* Service IP Family */ { - name: "same ServiceIPFamily", + name: "convert from ExternalName", tweakSvc: func(oldSvc, newSvc *core.Service) { - ipv4Service := core.IPv4Protocol - oldSvc.Spec.Type = core.ServiceTypeClusterIP - oldSvc.Spec.IPFamily = &ipv4Service - + oldSvc.Spec.Type = core.ServiceTypeExternalName newSvc.Spec.Type = core.ServiceTypeClusterIP - newSvc.Spec.IPFamily = &ipv4Service }, numErrs: 0, }, + { + name: "invalid: convert to ExternalName", + tweakSvc: func(oldSvc, newSvc *core.Service) { + singleStack := core.IPFamilyPolicySingleStack + + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.ClusterIP = "10.0.0.10" + oldSvc.Spec.ClusterIPs = []string{"10.0.0.10"} + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + oldSvc.Spec.IPFamilyPolicy = &singleStack + + newSvc.Spec.Type = core.ServiceTypeExternalName + newSvc.Spec.ExternalName = "foo" + /* + not removing these fields is a validation error + strategy takes care of resetting Families & Policy if ClusterIPs + were reset. But it does not get called in validation testing. + */ + newSvc.Spec.ClusterIP = "10.0.0.10" + newSvc.Spec.ClusterIPs = []string{"10.0.0.10"} + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + newSvc.Spec.IPFamilyPolicy = &singleStack + + }, + numErrs: 3, + }, + { + name: "valid: convert to ExternalName", + tweakSvc: func(oldSvc, newSvc *core.Service) { + singleStack := core.IPFamilyPolicySingleStack + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.ClusterIP = "10.0.0.10" + oldSvc.Spec.ClusterIPs = []string{"10.0.0.10"} + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + oldSvc.Spec.IPFamilyPolicy = &singleStack + + newSvc.Spec.Type = core.ServiceTypeExternalName + newSvc.Spec.ExternalName = "foo" + }, + numErrs: 0, + }, + + { + name: "same ServiceIPFamily", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "same ServiceIPFamily, change IPFamilyPolicy to singleStack", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = nil + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.IPFamilyPolicy = &singleStack + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "same ServiceIPFamily, change IPFamilyPolicy singleStack => requireDualStack", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &singleStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + + { + name: "add a new ServiceIPFamily", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { name: "ExternalName while changing Service IPFamily", tweakSvc: func(oldSvc, newSvc *core.Service) { - ipv4Service := core.IPv4Protocol oldSvc.Spec.ExternalName = "somename" oldSvc.Spec.Type = core.ServiceTypeExternalName - oldSvc.Spec.IPFamily = &ipv4Service - ipv6Service := core.IPv6Protocol newSvc.Spec.ExternalName = "somename" newSvc.Spec.Type = core.ServiceTypeExternalName - newSvc.Spec.IPFamily = &ipv6Service }, numErrs: 0, }, { name: "setting ipfamily from nil to v4", tweakSvc: func(oldSvc, newSvc *core.Service) { - oldSvc.Spec.IPFamily = nil + oldSvc.Spec.IPFamilies = nil - ipv4Service := core.IPv4Protocol newSvc.Spec.ExternalName = "somename" - newSvc.Spec.IPFamily = &ipv4Service + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} }, numErrs: 0, }, { name: "setting ipfamily from nil to v6", tweakSvc: func(oldSvc, newSvc *core.Service) { - oldSvc.Spec.IPFamily = nil + oldSvc.Spec.IPFamilies = nil - ipv6Service := core.IPv6Protocol newSvc.Spec.ExternalName = "somename" - newSvc.Spec.IPFamily = &ipv6Service + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} }, numErrs: 0, }, { - name: "remove ipfamily (covered by conditional validation)", + name: "change primary ServiceIPFamily", tweakSvc: func(oldSvc, newSvc *core.Service) { - ipv6Service := core.IPv6Protocol - oldSvc.Spec.IPFamily = &ipv6Service - - newSvc.Spec.IPFamily = nil - }, - numErrs: 0, - }, - - { - name: "change ServiceIPFamily (covered by conditional validation)", - tweakSvc: func(oldSvc, newSvc *core.Service) { - ipv4Service := core.IPv4Protocol + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} oldSvc.Spec.Type = core.ServiceTypeClusterIP - oldSvc.Spec.IPFamily = &ipv4Service + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} - ipv6Service := core.IPv6Protocol newSvc.Spec.Type = core.ServiceTypeClusterIP - newSvc.Spec.IPFamily = &ipv6Service + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + }, + numErrs: 2, + }, + /* upgrade + downgrade from/to dualstack tests */ + { + name: "valid: upgrade to dual stack with requiredDualStack", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &singleStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} }, numErrs: 0, }, + { + name: "valid: upgrade to dual stack with preferDualStack", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &singleStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.IPFamilyPolicy = &preferDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + + { + name: "valid: upgrade to dual stack, no specific secondary ip", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &singleStack + + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid: upgrade to dual stack, with specific secondary ip", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &singleStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, + }, + { + name: "valid: downgrade from dual to single", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.IPFamilyPolicy = &singleStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "valid: change families for a headless service", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "None" + oldSvc.Spec.ClusterIPs = []string{"None"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "None" + newSvc.Spec.ClusterIPs = []string{"None"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol, core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "valid: upgrade a headless service", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "None" + oldSvc.Spec.ClusterIPs = []string{"None"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &singleStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "None" + newSvc.Spec.ClusterIPs = []string{"None"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol, core.IPv4Protocol} + }, + numErrs: 0, + }, + { + name: "valid: downgrade a headless service", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "None" + oldSvc.Spec.ClusterIPs = []string{"None"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "None" + newSvc.Spec.ClusterIPs = []string{"None"} + newSvc.Spec.IPFamilyPolicy = &singleStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol} + }, + numErrs: 0, + }, + + { + name: "invalid flip families", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.40" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "2001::1" + newSvc.Spec.ClusterIPs = []string{"2001::1", "1.2.3.5"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv6Protocol, core.IPv4Protocol} + }, + numErrs: 4, + }, + { + name: "invalid change first ip, in dualstack service", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5", "2001::1"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 1, + }, + { + name: "invalid, change second ip in dualstack service", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2002::1"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 1, + }, + { + name: "downgrade keeping the families", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.IPFamilyPolicy = &singleStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 0, // families and ips are trimmed in strategy + }, + { + name: "invalid, downgrade without changing to singleStack", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.4" + newSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 2, + }, + { + name: "invalid, downgrade and change primary ip", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4", "2001::1"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &requireDualStack + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} + newSvc.Spec.IPFamilyPolicy = &singleStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + }, + numErrs: 1, + }, + { + name: "invalid: upgrade to dual stack and change primary", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.ClusterIP = "1.2.3.4" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + oldSvc.Spec.Type = core.ServiceTypeClusterIP + oldSvc.Spec.IPFamilyPolicy = &singleStack + + oldSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol} + + newSvc.Spec.Type = core.ServiceTypeClusterIP + newSvc.Spec.ClusterIP = "1.2.3.5" + newSvc.Spec.ClusterIPs = []string{"1.2.3.5"} + newSvc.Spec.IPFamilyPolicy = &requireDualStack + newSvc.Spec.IPFamilies = []core.IPFamily{core.IPv4Protocol, core.IPv6Protocol} + }, + numErrs: 1, + }, + { name: "update with valid app protocol, field unset, gate disabled", tweakSvc: func(oldSvc, newSvc *core.Service) { diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index b3e0a053a8b..c305761dbd9 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -5255,6 +5255,21 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { (*out)[key] = val } } + if in.ClusterIPs != nil { + in, out := &in.ClusterIPs, &out.ClusterIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.IPFamilies != nil { + in, out := &in.IPFamilies, &out.IPFamilies + *out = make([]IPFamily, len(*in)) + copy(*out, *in) + } + if in.IPFamilyPolicy != nil { + in, out := &in.IPFamilyPolicy, &out.IPFamilyPolicy + *out = new(IPFamilyPolicyType) + **out = **in + } if in.ExternalIPs != nil { in, out := &in.ExternalIPs, &out.ExternalIPs *out = make([]string, len(*in)) @@ -5270,11 +5285,6 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.IPFamily != nil { - in, out := &in.IPFamily, &out.IPFamily - *out = new(IPFamily) - **out = **in - } if in.TopologyKeys != nil { in, out := &in.TopologyKeys, &out.TopologyKeys *out = make([]string, len(*in)) diff --git a/pkg/controller/endpoint/endpoints_controller.go b/pkg/controller/endpoint/endpoints_controller.go index 9fdbbecd422..a1bc0f11980 100644 --- a/pkg/controller/endpoint/endpoints_controller.go +++ b/pkg/controller/endpoint/endpoints_controller.go @@ -216,22 +216,49 @@ func (e *Controller) addPod(obj interface{}) { func podToEndpointAddressForService(svc *v1.Service, pod *v1.Pod) (*v1.EndpointAddress, error) { var endpointIP string + ipFamily := v1.IPv4Protocol if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { // In a legacy cluster, the pod IP is guaranteed to be usable endpointIP = pod.Status.PodIP } else { - ipv6Service := endpointutil.IsIPv6Service(svc) + //feature flag enabled and pods may have multiple IPs + if len(svc.Spec.IPFamilies) > 0 { + // controller is connected to an api-server that correctly sets IPFamilies + ipFamily = svc.Spec.IPFamilies[0] // this works for headful and headless + } else { + // controller is connected to an api server that does not correctly + // set IPFamilies (e.g. old api-server during an upgrade) + if len(svc.Spec.ClusterIP) > 0 && svc.Spec.ClusterIP != v1.ClusterIPNone { + // headful service. detect via service clusterIP + if utilnet.IsIPv6String(svc.Spec.ClusterIP) { + ipFamily = v1.IPv6Protocol + } + } else { + // Since this is a headless service we use podIP to identify the family. + // This assumes that status.PodIP is assigned correctly (follows pod cidr and + // pod cidr list order is same as service cidr list order). The expectation is + // this is *most probably* the case. + + // if the family was incorrectly indentified then this will be corrected once the + // the upgrade is completed (controller connects to api-server that correctly defaults services) + if utilnet.IsIPv6String(pod.Status.PodIP) { + ipFamily = v1.IPv6Protocol + } + } + } + + // find an ip that matches the family for _, podIP := range pod.Status.PodIPs { - ipv6PodIP := utilnet.IsIPv6String(podIP.IP) - if ipv6Service == ipv6PodIP { + if (ipFamily == v1.IPv6Protocol) == utilnet.IsIPv6String(podIP.IP) { endpointIP = podIP.IP break } } - if endpointIP == "" { - return nil, fmt.Errorf("failed to find a matching endpoint for service %v", svc.Name) - } + } + + if endpointIP == "" { + return nil, fmt.Errorf("failed to find a matching endpoint for service %v", svc.Name) } return &v1.EndpointAddress{ diff --git a/pkg/controller/endpoint/endpoints_controller_test.go b/pkg/controller/endpoint/endpoints_controller_test.go index 3d6e6e59ae0..0da4dd1aefd 100644 --- a/pkg/controller/endpoint/endpoints_controller_test.go +++ b/pkg/controller/endpoint/endpoints_controller_test.go @@ -1256,8 +1256,8 @@ func TestPodToEndpointAddressForService(t *testing.T) { service: v1.Service{ Spec: v1.ServiceSpec{ - ClusterIP: v1.ClusterIPNone, - IPFamily: &ipv4, + ClusterIP: v1.ClusterIPNone, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, }, }, @@ -1289,7 +1289,7 @@ func TestPodToEndpointAddressForService(t *testing.T) { }, }, - expectedEndpointFamily: ipv4, + expectedEndpointFamily: ipv6, }, { name: "v6 service, in a dual stack cluster", @@ -1320,33 +1320,32 @@ func TestPodToEndpointAddressForService(t *testing.T) { expectedEndpointFamily: ipv6, }, { - name: "v6 headless service, in a dual stack cluster", + name: "v6 headless service, in a dual stack cluster (connected to a new api-server)", enableDualStack: true, ipFamilies: ipv4ipv6, service: v1.Service{ Spec: v1.ServiceSpec{ - ClusterIP: v1.ClusterIPNone, - IPFamily: &ipv6, + ClusterIP: v1.ClusterIPNone, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, // <- set by a api-server defaulting logic }, }, expectedEndpointFamily: ipv6, }, { - name: "v6 legacy headless service, in a dual stack cluster", + name: "v6 legacy headless service, in a dual stack cluster (connected to a old api-server)", enableDualStack: false, ipFamilies: ipv4ipv6, service: v1.Service{ Spec: v1.ServiceSpec{ - ClusterIP: v1.ClusterIPNone, + ClusterIP: v1.ClusterIPNone, // <- families are not set by api-server }, }, - // This is not the behavior we *want*, but it's the behavior we currently expect. expectedEndpointFamily: ipv4, }, diff --git a/pkg/controller/endpointslice/BUILD b/pkg/controller/endpointslice/BUILD index bd0d15b0d32..6bbe6bb7065 100644 --- a/pkg/controller/endpointslice/BUILD +++ b/pkg/controller/endpointslice/BUILD @@ -27,6 +27,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/pkg/controller/endpointslice/endpointslice_controller_test.go b/pkg/controller/endpointslice/endpointslice_controller_test.go index dd0cc024520..9c17c499c6d 100644 --- a/pkg/controller/endpointslice/endpointslice_controller_test.go +++ b/pkg/controller/endpointslice/endpointslice_controller_test.go @@ -340,7 +340,6 @@ func TestSyncServiceFull(t *testing.T) { client, esController := newController([]string{"node-1"}, time.Duration(0)) namespace := metav1.NamespaceDefault serviceName := "all-the-protocols" - ipv6Family := v1.IPv6Protocol pod1 := newPod(1, namespace, true, 0) pod1.Status.PodIPs = []v1.PodIP{{IP: "1.2.3.4"}} @@ -364,8 +363,8 @@ func TestSyncServiceFull(t *testing.T) { {Name: "udp-example", TargetPort: intstr.FromInt(161), Protocol: v1.ProtocolUDP}, {Name: "sctp-example", TargetPort: intstr.FromInt(3456), Protocol: v1.ProtocolSCTP}, }, - Selector: map[string]string{"foo": "bar"}, - IPFamily: &ipv6Family, + Selector: map[string]string{"foo": "bar"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, }, } esController.serviceStore.Add(service) @@ -491,8 +490,9 @@ func TestPodAddsBatching(t *testing.T) { esController.serviceStore.Add(&v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns}, Spec: v1.ServiceSpec{ - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{Port: 80}}, + Selector: map[string]string{"foo": "bar"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + Ports: []v1.ServicePort{{Port: 80}}, }, }) @@ -624,8 +624,9 @@ func TestPodUpdatesBatching(t *testing.T) { esController.serviceStore.Add(&v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns}, Spec: v1.ServiceSpec{ - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{Port: 80}}, + Selector: map[string]string{"foo": "bar"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + Ports: []v1.ServicePort{{Port: 80}}, }, }) @@ -758,8 +759,9 @@ func TestPodDeleteBatching(t *testing.T) { esController.serviceStore.Add(&v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns}, Spec: v1.ServiceSpec{ - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{Port: 80}}, + Selector: map[string]string{"foo": "bar"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + Ports: []v1.ServicePort{{Port: 80}}, }, }) @@ -810,8 +812,9 @@ func createService(t *testing.T, esController *endpointSliceController, namespac UID: types.UID(namespace + "-" + serviceName), }, Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{{TargetPort: intstr.FromInt(80)}}, - Selector: map[string]string{"foo": "bar"}, + Ports: []v1.ServicePort{{TargetPort: intstr.FromInt(80)}}, + Selector: map[string]string{"foo": "bar"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, }, } esController.serviceStore.Add(service) diff --git a/pkg/controller/endpointslice/reconciler.go b/pkg/controller/endpointslice/reconciler.go index 01b4e689c96..147c62d4c8b 100644 --- a/pkg/controller/endpointslice/reconciler.go +++ b/pkg/controller/endpointslice/reconciler.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" corelisters "k8s.io/client-go/listers/core/v1" @@ -56,12 +57,62 @@ type endpointMeta struct { // slices for the given service. It creates, updates, or deletes endpoint slices // to ensure the desired set of pods are represented by endpoint slices. func (r *reconciler) reconcile(service *corev1.Service, pods []*corev1.Pod, existingSlices []*discovery.EndpointSlice, triggerTime time.Time) error { - addressType := discovery.AddressTypeIPv4 + slicesToDelete := []*discovery.EndpointSlice{} // slices that are no longer matching any address the service has + errs := []error{} // all errors generated in the process of reconciling + slicesByAddressType := make(map[discovery.AddressType][]*discovery.EndpointSlice) // slices by address type - if endpointutil.IsIPv6Service(service) { - addressType = discovery.AddressTypeIPv6 + // addresses that this service supports [o(1) find] + serviceSupportedAddressesTypes := getAddressTypesForService(service) + + // loop through slices identifying their address type. + // slices that no longer match address type supported by services + // go to delete, other slices goes to the reconciler machinery + // for further adjustment + for _, existingSlice := range existingSlices { + // service no longer supports that address type, add it to deleted slices + if _, ok := serviceSupportedAddressesTypes[existingSlice.AddressType]; !ok { + slicesToDelete = append(slicesToDelete, existingSlice) + continue + } + + // add list if it is not on our map + if _, ok := slicesByAddressType[existingSlice.AddressType]; !ok { + slicesByAddressType[existingSlice.AddressType] = make([]*discovery.EndpointSlice, 0, 1) + } + + slicesByAddressType[existingSlice.AddressType] = append(slicesByAddressType[existingSlice.AddressType], existingSlice) } + // reconcile for existing. + for addressType := range serviceSupportedAddressesTypes { + existingSlices := slicesByAddressType[addressType] + err := r.reconcileByAddressType(service, pods, existingSlices, triggerTime, addressType) + if err != nil { + errs = append(errs, err) + } + } + + // delete those which are of addressType that is no longer supported + // by the service + for _, sliceToDelete := range slicesToDelete { + err := r.client.DiscoveryV1beta1().EndpointSlices(service.Namespace).Delete(context.TODO(), sliceToDelete.Name, metav1.DeleteOptions{}) + if err != nil { + errs = append(errs, fmt.Errorf("Error deleting %s EndpointSlice for Service %s/%s: %v", sliceToDelete.Name, service.Namespace, service.Name, err)) + } else { + r.endpointSliceTracker.Delete(sliceToDelete) + metrics.EndpointSliceChanges.WithLabelValues("delete").Inc() + } + } + + return utilerrors.NewAggregate(errs) +} + +// reconcileByAddressType takes a set of pods currently matching a service selector and +// compares them with the endpoints already present in any existing endpoint +// slices (by address type) for the given service. It creates, updates, or deletes endpoint slices +// to ensure the desired set of pods are represented by endpoint slices. +func (r *reconciler) reconcileByAddressType(service *corev1.Service, pods []*corev1.Pod, existingSlices []*discovery.EndpointSlice, triggerTime time.Time, addressType discovery.AddressType) error { + slicesToCreate := []*discovery.EndpointSlice{} slicesToUpdate := []*discovery.EndpointSlice{} slicesToDelete := []*discovery.EndpointSlice{} @@ -70,7 +121,7 @@ func (r *reconciler) reconcile(service *corev1.Service, pods []*corev1.Pod, exis existingSlicesByPortMap := map[endpointutil.PortMapKey][]*discovery.EndpointSlice{} numExistingEndpoints := 0 for _, existingSlice := range existingSlices { - if existingSlice.AddressType == addressType && ownedBy(existingSlice, service) { + if ownedBy(existingSlice, service) { epHash := endpointutil.NewPortMapKey(existingSlice.Ports) existingSlicesByPortMap[epHash] = append(existingSlicesByPortMap[epHash], existingSlice) numExistingEndpoints += len(existingSlice.Endpoints) @@ -106,7 +157,7 @@ func (r *reconciler) reconcile(service *corev1.Service, pods []*corev1.Pod, exis if err != nil { return err } - endpoint := podToEndpoint(pod, node, service) + endpoint := podToEndpoint(pod, node, service, addressType) if len(endpoint.Addresses) > 0 { desiredEndpointsByPortMap[epHash].Insert(&endpoint) numDesiredEndpoints++ diff --git a/pkg/controller/endpointslice/reconciler_test.go b/pkg/controller/endpointslice/reconciler_test.go index 7375f9a3ec6..4710348712b 100644 --- a/pkg/controller/endpointslice/reconciler_test.go +++ b/pkg/controller/endpointslice/reconciler_test.go @@ -70,7 +70,10 @@ func TestReconcileEmpty(t *testing.T) { // a slice should be created func TestReconcile1Pod(t *testing.T) { namespace := "test" - ipv6Family := corev1.IPv6Protocol + noFamilyService, _ := newServiceAndEndpointMeta("foo", namespace) + noFamilyService.Spec.ClusterIP = "10.0.0.10" + noFamilyService.Spec.IPFamilies = nil + svcv4, _ := newServiceAndEndpointMeta("foo", namespace) svcv4ClusterIP, _ := newServiceAndEndpointMeta("foo", namespace) svcv4ClusterIP.Spec.ClusterIP = "1.1.1.1" @@ -80,9 +83,17 @@ func TestReconcile1Pod(t *testing.T) { svcv4BadLabels.Labels = map[string]string{discovery.LabelServiceName: "bad", discovery.LabelManagedBy: "actor", corev1.IsHeadlessService: "invalid"} svcv6, _ := newServiceAndEndpointMeta("foo", namespace) - svcv6.Spec.IPFamily = &ipv6Family + svcv6.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv6Protocol} svcv6ClusterIP, _ := newServiceAndEndpointMeta("foo", namespace) svcv6ClusterIP.Spec.ClusterIP = "1234::5678:0000:0000:9abc:def1" + // newServiceAndEndpointMeta generates v4 single stack + svcv6ClusterIP.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv6Protocol} + + // dual stack + dualStackSvc, _ := newServiceAndEndpointMeta("foo", namespace) + dualStackSvc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + dualStackSvc.Spec.ClusterIP = "10.0.0.10" + dualStackSvc.Spec.ClusterIPs = []string{"10.0.0.10", "2000::1"} pod1 := newPod(1, namespace, true, 1) pod1.Status.PodIPs = []corev1.PodIP{{IP: "1.2.3.4"}, {IP: "1234::5678:0000:0000:9abc:def0"}} @@ -98,26 +109,56 @@ func TestReconcile1Pod(t *testing.T) { } testCases := map[string]struct { - service corev1.Service - expectedAddressType discovery.AddressType - expectedEndpoint discovery.Endpoint - expectedLabels map[string]string + service corev1.Service + expectedAddressType discovery.AddressType + expectedEndpoint discovery.Endpoint + expectedLabels map[string]string + expectedEndpointPerSlice map[discovery.AddressType][]discovery.Endpoint }{ - "ipv4": { - service: svcv4, - expectedAddressType: discovery.AddressTypeIPv4, - expectedEndpoint: discovery.Endpoint{ - Addresses: []string{"1.2.3.4"}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, - Topology: map[string]string{ - "kubernetes.io/hostname": "node-1", - "topology.kubernetes.io/zone": "us-central1-a", - "topology.kubernetes.io/region": "us-central1", + "no-family-service": { + service: noFamilyService, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv4: { + { + Addresses: []string{"1.2.3.4"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, }, - TargetRef: &corev1.ObjectReference{ - Kind: "Pod", - Namespace: namespace, - Name: "pod1", + }, + expectedLabels: map[string]string{ + discovery.LabelManagedBy: controllerName, + discovery.LabelServiceName: "foo", + }, + }, + + "ipv4": { + service: svcv4, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv4: { + { + Addresses: []string{"1.2.3.4"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, }, }, expectedLabels: map[string]string{ @@ -127,7 +168,25 @@ func TestReconcile1Pod(t *testing.T) { }, }, "ipv4-clusterip": { - service: svcv4ClusterIP, + service: svcv4ClusterIP, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv4: { + { + Addresses: []string{"1.2.3.4"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, + }, + }, expectedAddressType: discovery.AddressTypeIPv4, expectedEndpoint: discovery.Endpoint{ Addresses: []string{"1.2.3.4"}, @@ -149,7 +208,25 @@ func TestReconcile1Pod(t *testing.T) { }, }, "ipv4-labels": { - service: svcv4Labels, + service: svcv4Labels, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv4: { + { + Addresses: []string{"1.2.3.4"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, + }, + }, expectedAddressType: discovery.AddressTypeIPv4, expectedEndpoint: discovery.Endpoint{ Addresses: []string{"1.2.3.4"}, @@ -173,7 +250,25 @@ func TestReconcile1Pod(t *testing.T) { }, }, "ipv4-bad-labels": { - service: svcv4BadLabels, + service: svcv4BadLabels, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv4: { + { + Addresses: []string{"1.2.3.4"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, + }, + }, expectedAddressType: discovery.AddressTypeIPv4, expectedEndpoint: discovery.Endpoint{ Addresses: []string{"1.2.3.4"}, @@ -195,21 +290,25 @@ func TestReconcile1Pod(t *testing.T) { corev1.IsHeadlessService: "", }, }, + "ipv6": { - service: svcv6, - expectedAddressType: discovery.AddressTypeIPv6, - expectedEndpoint: discovery.Endpoint{ - Addresses: []string{"1234::5678:0000:0000:9abc:def0"}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, - Topology: map[string]string{ - "kubernetes.io/hostname": "node-1", - "topology.kubernetes.io/zone": "us-central1-a", - "topology.kubernetes.io/region": "us-central1", - }, - TargetRef: &corev1.ObjectReference{ - Kind: "Pod", - Namespace: namespace, - Name: "pod1", + service: svcv6, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv6: { + { + Addresses: []string{"1234::5678:0000:0000:9abc:def0"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, }, }, expectedLabels: map[string]string{ @@ -218,21 +317,67 @@ func TestReconcile1Pod(t *testing.T) { corev1.IsHeadlessService: "", }, }, + "ipv6-clusterip": { - service: svcv6ClusterIP, - expectedAddressType: discovery.AddressTypeIPv6, - expectedEndpoint: discovery.Endpoint{ - Addresses: []string{"1234::5678:0000:0000:9abc:def0"}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, - Topology: map[string]string{ - "kubernetes.io/hostname": "node-1", - "topology.kubernetes.io/zone": "us-central1-a", - "topology.kubernetes.io/region": "us-central1", + service: svcv6ClusterIP, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv6: { + { + Addresses: []string{"1234::5678:0000:0000:9abc:def0"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, }, - TargetRef: &corev1.ObjectReference{ - Kind: "Pod", - Namespace: namespace, - Name: "pod1", + }, + expectedLabels: map[string]string{ + discovery.LabelManagedBy: controllerName, + discovery.LabelServiceName: "foo", + }, + }, + + "dualstack-service": { + service: dualStackSvc, + expectedEndpointPerSlice: map[discovery.AddressType][]discovery.Endpoint{ + discovery.AddressTypeIPv6: { + { + Addresses: []string{"1234::5678:0000:0000:9abc:def0"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, + }, + discovery.AddressTypeIPv4: { + { + Addresses: []string{"1.2.3.4"}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + Topology: map[string]string{ + "kubernetes.io/hostname": "node-1", + "topology.kubernetes.io/zone": "us-central1-a", + "topology.kubernetes.io/region": "us-central1", + }, + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Namespace: namespace, + Name: "pod1", + }, + }, }, }, expectedLabels: map[string]string{ @@ -250,41 +395,61 @@ func TestReconcile1Pod(t *testing.T) { r := newReconciler(client, []*corev1.Node{node1}, defaultMaxEndpointsPerSlice) reconcileHelper(t, r, &testCase.service, []*corev1.Pod{pod1}, []*discovery.EndpointSlice{}, triggerTime) - if len(client.Actions()) != 1 { - t.Errorf("Expected 1 clientset action, got %d", len(client.Actions())) + if len(client.Actions()) != len(testCase.expectedEndpointPerSlice) { + t.Errorf("Expected %v clientset action, got %d", len(testCase.expectedEndpointPerSlice), len(client.Actions())) } slices := fetchEndpointSlices(t, client, namespace) - if len(slices) != 1 { - t.Fatalf("Expected 1 EndpointSlice, got %d", len(slices)) + if len(slices) != len(testCase.expectedEndpointPerSlice) { + t.Fatalf("Expected %v EndpointSlice, got %d", len(testCase.expectedEndpointPerSlice), len(slices)) } - slice := slices[0] - if !strings.HasPrefix(slice.Name, testCase.service.Name) { - t.Errorf("Expected EndpointSlice name to start with %s, got %s", testCase.service.Name, slice.Name) + for _, slice := range slices { + if !strings.HasPrefix(slice.Name, testCase.service.Name) { + t.Fatalf("Expected EndpointSlice name to start with %s, got %s", testCase.service.Name, slice.Name) + } + + if !reflect.DeepEqual(testCase.expectedLabels, slice.Labels) { + t.Errorf("Expected EndpointSlice to have labels: %v , got %v", testCase.expectedLabels, slice.Labels) + } + if slice.Labels[discovery.LabelServiceName] != testCase.service.Name { + t.Fatalf("Expected EndpointSlice to have label set with %s value, got %s", testCase.service.Name, slice.Labels[discovery.LabelServiceName]) + } + + if slice.Annotations[corev1.EndpointsLastChangeTriggerTime] != triggerTime.Format(time.RFC3339Nano) { + t.Fatalf("Expected EndpointSlice trigger time annotation to be %s, got %s", triggerTime.Format(time.RFC3339Nano), slice.Annotations[corev1.EndpointsLastChangeTriggerTime]) + } + + // validate that this slice has address type matching expected + expectedEndPointList := testCase.expectedEndpointPerSlice[slice.AddressType] + if expectedEndPointList == nil { + t.Fatalf("address type %v is not expected", slice.AddressType) + } + + if len(slice.Endpoints) != len(expectedEndPointList) { + t.Fatalf("Expected %v Endpoint, got %d", len(expectedEndPointList), len(slice.Endpoints)) + } + + // test is limited to *ONE* endpoint + endpoint := slice.Endpoints[0] + if !reflect.DeepEqual(endpoint, expectedEndPointList[0]) { + t.Fatalf("Expected endpoint: %+v, got: %+v", expectedEndPointList[0], endpoint) + } + + expectTrackedResourceVersion(t, r.endpointSliceTracker, &slice, "100") + + expectMetrics(t, + expectedMetrics{ + desiredSlices: 1, + actualSlices: 1, + desiredEndpoints: 1, + addedPerSync: len(testCase.expectedEndpointPerSlice), + removedPerSync: 0, + numCreated: len(testCase.expectedEndpointPerSlice), + numUpdated: 0, + numDeleted: 0}) } - - if !reflect.DeepEqual(testCase.expectedLabels, slice.Labels) { - t.Errorf("Expected EndpointSlice to have labels: %v , got %v", testCase.expectedLabels, slice.Labels) - } - - if slice.Annotations[corev1.EndpointsLastChangeTriggerTime] != triggerTime.Format(time.RFC3339Nano) { - t.Errorf("Expected EndpointSlice trigger time annotation to be %s, got %s", triggerTime.Format(time.RFC3339Nano), slice.Annotations[corev1.EndpointsLastChangeTriggerTime]) - } - - if len(slice.Endpoints) != 1 { - t.Fatalf("Expected 1 Endpoint, got %d", len(slice.Endpoints)) - } - - endpoint := slice.Endpoints[0] - if !reflect.DeepEqual(endpoint, testCase.expectedEndpoint) { - t.Errorf("Expected endpoint: %+v, got: %+v", testCase.expectedEndpoint, endpoint) - } - - expectTrackedResourceVersion(t, r.endpointSliceTracker, &slice, "100") - - expectMetrics(t, expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 1, addedPerSync: 1, removedPerSync: 0, numCreated: 1, numUpdated: 0, numDeleted: 0}) }) } } @@ -404,13 +569,13 @@ func TestReconcileEndpointSlicesSomePreexisting(t *testing.T) { // have approximately 1/4 in first slice endpointSlice1 := newEmptyEndpointSlice(1, namespace, endpointMeta, svc) for i := 1; i < len(pods)-4; i += 4 { - endpointSlice1.Endpoints = append(endpointSlice1.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc)) + endpointSlice1.Endpoints = append(endpointSlice1.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc, discovery.AddressTypeIPv4)) } // have approximately 1/4 in second slice endpointSlice2 := newEmptyEndpointSlice(2, namespace, endpointMeta, svc) for i := 3; i < len(pods)-4; i += 4 { - endpointSlice2.Endpoints = append(endpointSlice2.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc)) + endpointSlice2.Endpoints = append(endpointSlice2.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc, discovery.AddressTypeIPv4)) } existingSlices := []*discovery.EndpointSlice{endpointSlice1, endpointSlice2} @@ -460,13 +625,13 @@ func TestReconcileEndpointSlicesSomePreexistingWorseAllocation(t *testing.T) { // have approximately 1/4 in first slice endpointSlice1 := newEmptyEndpointSlice(1, namespace, endpointMeta, svc) for i := 1; i < len(pods)-4; i += 4 { - endpointSlice1.Endpoints = append(endpointSlice1.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc)) + endpointSlice1.Endpoints = append(endpointSlice1.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc, discovery.AddressTypeIPv4)) } // have approximately 1/4 in second slice endpointSlice2 := newEmptyEndpointSlice(2, namespace, endpointMeta, svc) for i := 3; i < len(pods)-4; i += 4 { - endpointSlice2.Endpoints = append(endpointSlice2.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc)) + endpointSlice2.Endpoints = append(endpointSlice2.Endpoints, podToEndpoint(pods[i], &corev1.Node{}, &svc, discovery.AddressTypeIPv4)) } existingSlices := []*discovery.EndpointSlice{endpointSlice1, endpointSlice2} @@ -621,7 +786,7 @@ func TestReconcileEndpointSlicesRecycling(t *testing.T) { if i%30 == 0 { existingSlices = append(existingSlices, newEmptyEndpointSlice(sliceNum, namespace, endpointMeta, svc)) } - existingSlices[sliceNum].Endpoints = append(existingSlices[sliceNum].Endpoints, podToEndpoint(pod, &corev1.Node{}, &svc)) + existingSlices[sliceNum].Endpoints = append(existingSlices[sliceNum].Endpoints, podToEndpoint(pod, &corev1.Node{}, &svc, discovery.AddressTypeIPv4)) } cmc := newCacheMutationCheck(existingSlices) @@ -663,7 +828,7 @@ func TestReconcileEndpointSlicesUpdatePacking(t *testing.T) { slice1 := newEmptyEndpointSlice(1, namespace, endpointMeta, svc) for i := 0; i < 80; i++ { pod := newPod(i, namespace, true, 1) - slice1.Endpoints = append(slice1.Endpoints, podToEndpoint(pod, &corev1.Node{}, &svc)) + slice1.Endpoints = append(slice1.Endpoints, podToEndpoint(pod, &corev1.Node{}, &svc, discovery.AddressTypeIPv4)) pods = append(pods, pod) } existingSlices = append(existingSlices, slice1) @@ -671,7 +836,7 @@ func TestReconcileEndpointSlicesUpdatePacking(t *testing.T) { slice2 := newEmptyEndpointSlice(2, namespace, endpointMeta, svc) for i := 100; i < 120; i++ { pod := newPod(i, namespace, true, 1) - slice2.Endpoints = append(slice2.Endpoints, podToEndpoint(pod, &corev1.Node{}, &svc)) + slice2.Endpoints = append(slice2.Endpoints, podToEndpoint(pod, &corev1.Node{}, &svc, discovery.AddressTypeIPv4)) pods = append(pods, pod) } existingSlices = append(existingSlices, slice2) @@ -724,7 +889,7 @@ func TestReconcileEndpointSlicesReplaceDeprecated(t *testing.T) { slice1 := newEmptyEndpointSlice(1, namespace, endpointMeta, svc) for i := 0; i < 80; i++ { pod := newPod(i, namespace, true, 1) - slice1.Endpoints = append(slice1.Endpoints, podToEndpoint(pod, &corev1.Node{}, &corev1.Service{Spec: corev1.ServiceSpec{}})) + slice1.Endpoints = append(slice1.Endpoints, podToEndpoint(pod, &corev1.Node{}, &corev1.Service{Spec: corev1.ServiceSpec{}}, discovery.AddressTypeIPv4)) pods = append(pods, pod) } existingSlices = append(existingSlices, slice1) @@ -732,7 +897,7 @@ func TestReconcileEndpointSlicesReplaceDeprecated(t *testing.T) { slice2 := newEmptyEndpointSlice(2, namespace, endpointMeta, svc) for i := 100; i < 150; i++ { pod := newPod(i, namespace, true, 1) - slice2.Endpoints = append(slice2.Endpoints, podToEndpoint(pod, &corev1.Node{}, &corev1.Service{Spec: corev1.ServiceSpec{}})) + slice2.Endpoints = append(slice2.Endpoints, podToEndpoint(pod, &corev1.Node{}, &corev1.Service{Spec: corev1.ServiceSpec{}}, discovery.AddressTypeIPv4)) pods = append(pods, pod) } existingSlices = append(existingSlices, slice2) @@ -791,7 +956,7 @@ func TestReconcileEndpointSlicesRecreation(t *testing.T) { slice := newEmptyEndpointSlice(1, namespace, endpointMeta, svc) pod := newPod(1, namespace, true, 1) - slice.Endpoints = append(slice.Endpoints, podToEndpoint(pod, &corev1.Node{}, &corev1.Service{Spec: corev1.ServiceSpec{}})) + slice.Endpoints = append(slice.Endpoints, podToEndpoint(pod, &corev1.Node{}, &corev1.Service{Spec: corev1.ServiceSpec{}}, discovery.AddressTypeIPv4)) if !tc.ownedByService { slice.OwnerReferences[0].UID = "different" @@ -848,7 +1013,8 @@ func TestReconcileEndpointSlicesNamedPorts(t *testing.T) { TargetPort: portNameIntStr, Protocol: corev1.ProtocolTCP, }}, - Selector: map[string]string{"foo": "bar"}, + Selector: map[string]string{"foo": "bar"}, + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, }, } @@ -1221,6 +1387,11 @@ func expectUnorderedSlicesWithTopLevelAttrs(t *testing.T, endpointSlices []disco func expectActions(t *testing.T, actions []k8stesting.Action, num int, verb, resource string) { t.Helper() + // if actions are less the below logic will panic + if num > len(actions) { + t.Fatalf("len of actions %v is unexpected. Expected to be at least %v", len(actions), num+1) + } + for i := 0; i < num; i++ { relativePos := len(actions) - i - 1 assert.Equal(t, verb, actions[relativePos].GetVerb(), "Expected action -%d verb to be %s", i, verb) diff --git a/pkg/controller/endpointslice/utils.go b/pkg/controller/endpointslice/utils.go index 9ab78a0a11b..83cfc028633 100644 --- a/pkg/controller/endpointslice/utils.go +++ b/pkg/controller/endpointslice/utils.go @@ -37,8 +37,8 @@ import ( utilnet "k8s.io/utils/net" ) -// podToEndpoint returns an Endpoint object generated from pod, node, and service. -func podToEndpoint(pod *corev1.Pod, node *corev1.Node, service *corev1.Service) discovery.Endpoint { +// podToEndpoint returns an Endpoint object generated from a Pod, a Node, and a Service for a particular addressType. +func podToEndpoint(pod *corev1.Pod, node *corev1.Node, service *corev1.Service, addressType discovery.AddressType) discovery.Endpoint { // Build out topology information. This is currently limited to hostname, // zone, and region, but this will be expanded in the future. topology := map[string]string{} @@ -62,7 +62,7 @@ func podToEndpoint(pod *corev1.Pod, node *corev1.Node, service *corev1.Service) ready := service.Spec.PublishNotReadyAddresses || podutil.IsPodReady(pod) ep := discovery.Endpoint{ - Addresses: getEndpointAddresses(pod.Status, service), + Addresses: getEndpointAddresses(pod.Status, service, addressType), Conditions: discovery.EndpointConditions{ Ready: &ready, }, @@ -117,12 +117,16 @@ func getEndpointPorts(service *corev1.Service, pod *corev1.Pod) []discovery.Endp } // getEndpointAddresses returns a list of addresses generated from a pod status. -func getEndpointAddresses(podStatus corev1.PodStatus, service *corev1.Service) []string { +func getEndpointAddresses(podStatus corev1.PodStatus, service *corev1.Service, addressType discovery.AddressType) []string { addresses := []string{} for _, podIP := range podStatus.PodIPs { isIPv6PodIP := utilnet.IsIPv6String(podIP.IP) - if isIPv6PodIP == endpointutil.IsIPv6Service(service) { + if isIPv6PodIP && addressType == discovery.AddressTypeIPv6 { + addresses = append(addresses, podIP.IP) + } + + if !isIPv6PodIP && addressType == discovery.AddressTypeIPv4 { addresses = append(addresses, podIP.IP) } } @@ -346,3 +350,62 @@ func (sl endpointSliceEndpointLen) Swap(i, j int) { sl[i], sl[j] = sl[j], sl[i] func (sl endpointSliceEndpointLen) Less(i, j int) bool { return len(sl[i].Endpoints) > len(sl[j].Endpoints) } + +// returns a map of address types used by a service +func getAddressTypesForService(service *corev1.Service) map[discovery.AddressType]struct{} { + serviceSupportedAddresses := make(map[discovery.AddressType]struct{}) + // TODO: (khenidak) when address types are removed in favor of + // v1.IPFamily this will need to be removed, and work directly with + // v1.IPFamily types + + // IMPORTANT: we assume that IP of (discovery.AddressType enum) is never in use + // as it gets deprecated + for _, family := range service.Spec.IPFamilies { + if family == corev1.IPv4Protocol { + serviceSupportedAddresses[discovery.AddressTypeIPv4] = struct{}{} + } + + if family == corev1.IPv6Protocol { + serviceSupportedAddresses[discovery.AddressTypeIPv6] = struct{}{} + } + } + + if len(serviceSupportedAddresses) > 0 { + return serviceSupportedAddresses // we have found families for this service + } + + // TODO (khenidak) remove when (1) dual stack becomes + // enabled by default (2) v1.19 falls off supported versions + + // Why do we need this: + // a cluster being upgraded to the new apis + // will have service.spec.IPFamilies: nil + // if the controller manager connected to old api + // server. This will have the nasty side effect of + // removing all slices already created for this service. + // this will disable all routing to service vip (ClusterIP) + // this ensures that this does not happen. Same for headless services + // we assume it is dual stack, until they get defaulted by *new* api-server + // this ensures that traffic is not disrupted until then. But *may* + // include undesired families for headless services until then. + + if len(service.Spec.ClusterIP) > 0 && service.Spec.ClusterIP != corev1.ClusterIPNone { // headfull + addrType := discovery.AddressTypeIPv4 + if utilnet.IsIPv6String(service.Spec.ClusterIP) { + addrType = discovery.AddressTypeIPv6 + } + serviceSupportedAddresses[addrType] = struct{}{} + klog.V(2).Infof("couldn't find ipfamilies for headless service: %v/%v. This could happen if controller manager is connected to an old apiserver that does not support ip families yet. EndpointSlices for this Service will use %s as the IP Family based on familyOf(ClusterIP:%v).", service.Namespace, service.Name, addrType, service.Spec.ClusterIP) + return serviceSupportedAddresses + } + + // headless + // for now we assume two families. This should have minimal side effect + // if the service is headless with no selector, then this will remain the case + // if the service is headless with selector then chances are pods are still using single family + // since kubelet will need to restart in order to start patching pod status with multiple ips + serviceSupportedAddresses[discovery.AddressTypeIPv4] = struct{}{} + serviceSupportedAddresses[discovery.AddressTypeIPv6] = struct{}{} + klog.V(2).Infof("couldn't find ipfamilies for headless service: %v/%v likely because controller manager is likely connected to an old apiserver that does not support ip families yet. The service endpoint slice will use dual stack families until api-server default it correctly", service.Namespace, service.Name) + return serviceSupportedAddresses +} diff --git a/pkg/controller/endpointslice/utils_test.go b/pkg/controller/endpointslice/utils_test.go index 349d9f893d5..a84c8caa37d 100644 --- a/pkg/controller/endpointslice/utils_test.go +++ b/pkg/controller/endpointslice/utils_test.go @@ -385,7 +385,7 @@ func TestPodToEndpoint(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - endpoint := podToEndpoint(testCase.pod, testCase.node, testCase.svc) + endpoint := podToEndpoint(testCase.pod, testCase.node, testCase.svc, discovery.AddressTypeIPv4) if !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) { t.Errorf("Expected endpoint: %v, got: %v", testCase.expectedEndpoint, endpoint) } @@ -889,7 +889,8 @@ func newServiceAndEndpointMeta(name, namespace string) (v1.Service, endpointMeta Protocol: v1.ProtocolTCP, Name: name, }}, - Selector: map[string]string{"foo": "bar"}, + Selector: map[string]string{"foo": "bar"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, }, } @@ -918,3 +919,135 @@ func newEmptyEndpointSlice(n int, namespace string, endpointMeta endpointMeta, s Endpoints: []discovery.Endpoint{}, } } + +func TestSupportedServiceAddressType(t *testing.T) { + testCases := []struct { + name string + service v1.Service + expectedAddressTypes []discovery.AddressType + }{ + { + name: "v4 service with no ip families (cluster upgrade)", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv4}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: "10.0.0.10", + IPFamilies: nil, + }, + }, + }, + { + name: "v6 service with no ip families (cluster upgrade)", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv6}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: "2000::1", + IPFamilies: nil, + }, + }, + }, + { + name: "v4 service", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv4}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + }, + { + name: "v6 services", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv6}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, + }, + }, + { + name: "v4,v6 service", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv4, discovery.AddressTypeIPv6}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, + }, + }, + { + name: "v6,v4 service", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv6, discovery.AddressTypeIPv4}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + }, + }, + }, + { + name: "headless with no selector and no families (old api-server)", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv6, discovery.AddressTypeIPv4}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: v1.ClusterIPNone, + IPFamilies: nil, + }, + }, + }, + { + name: "headless with selector and no families (old api-server)", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv6, discovery.AddressTypeIPv4}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + Selector: map[string]string{"foo": "bar"}, + ClusterIP: v1.ClusterIPNone, + IPFamilies: nil, + }, + }, + }, + + { + name: "headless with no selector with families", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv4, discovery.AddressTypeIPv6}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: v1.ClusterIPNone, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, + }, + }, + { + name: "headless with selector with families", + expectedAddressTypes: []discovery.AddressType{discovery.AddressTypeIPv4, discovery.AddressTypeIPv6}, + service: v1.Service{ + Spec: v1.ServiceSpec{ + Selector: map[string]string{"foo": "bar"}, + ClusterIP: v1.ClusterIPNone, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + addressTypes := getAddressTypesForService(&testCase.service) + if len(addressTypes) != len(testCase.expectedAddressTypes) { + t.Fatalf("expected count address types %v got %v", len(testCase.expectedAddressTypes), len(addressTypes)) + } + + // compare + for _, expectedAddressType := range testCase.expectedAddressTypes { + found := false + for key := range addressTypes { + if key == expectedAddressType { + found = true + break + + } + } + if !found { + t.Fatalf("expected address type %v was not found in the result", expectedAddressType) + } + } + }) + } +} diff --git a/pkg/controller/util/endpoint/BUILD b/pkg/controller/util/endpoint/BUILD index 81f4068279b..3a17c9c71f9 100644 --- a/pkg/controller/util/endpoint/BUILD +++ b/pkg/controller/util/endpoint/BUILD @@ -10,7 +10,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/api/v1/pod:go_default_library", - "//pkg/apis/core/v1/helper:go_default_library", "//pkg/controller:go_default_library", "//pkg/util/hash:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -20,7 +19,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/pkg/controller/util/endpoint/controller_utils.go b/pkg/controller/util/endpoint/controller_utils.go index 9c2ab1eedf9..d1744441bec 100644 --- a/pkg/controller/util/endpoint/controller_utils.go +++ b/pkg/controller/util/endpoint/controller_utils.go @@ -32,10 +32,8 @@ import ( v1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" podutil "k8s.io/kubernetes/pkg/api/v1/pod" - "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/hash" - utilnet "k8s.io/utils/net" ) // ServiceSelectorCache is a cache of service selectors to avoid high CPU consumption caused by frequent calls to AsSelectorPreValidated (see #73527) @@ -277,18 +275,3 @@ func (sl portsInOrder) Less(i, j int) bool { h2 := DeepHashObjectToString(sl[j]) return h1 < h2 } - -// IsIPv6Service checks if svc should have IPv6 endpoints -func IsIPv6Service(svc *v1.Service) bool { - if helper.IsServiceIPSet(svc) { - return utilnet.IsIPv6String(svc.Spec.ClusterIP) - } else if svc.Spec.IPFamily != nil { - return *svc.Spec.IPFamily == v1.IPv6Protocol - } else { - // FIXME: for legacy headless Services with no IPFamily, the current - // thinking is that we should use the cluster default. Unfortunately - // the endpoint controller doesn't know the cluster default. For now, - // assume it's IPv4. - return false - } -} diff --git a/pkg/controlplane/controller.go b/pkg/controlplane/controller.go index ccdf12cd9f4..fc3bbbee7ad 100644 --- a/pkg/controlplane/controller.go +++ b/pkg/controlplane/controller.go @@ -292,6 +292,7 @@ func (c *Controller) CreateOrUpdateMasterServiceIfNeeded(serviceName string, ser } return nil } + singleStack := corev1.IPFamilyPolicySingleStack svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, @@ -303,6 +304,7 @@ func (c *Controller) CreateOrUpdateMasterServiceIfNeeded(serviceName string, ser // maintained by this code, not by the pod selector Selector: nil, ClusterIP: serviceIP.String(), + IPFamilyPolicy: &singleStack, SessionAffinity: corev1.ServiceAffinityNone, Type: serviceType, }, diff --git a/pkg/controlplane/controller_test.go b/pkg/controlplane/controller_test.go index 8b68e958000..0b07a9c3558 100644 --- a/pkg/controlplane/controller_test.go +++ b/pkg/controlplane/controller_test.go @@ -592,6 +592,7 @@ func TestEmptySubsets(t *testing.T) { } func TestCreateOrUpdateMasterService(t *testing.T) { + singleStack := corev1.IPFamilyPolicySingleStack ns := metav1.NamespaceDefault om := func(name string) metav1.ObjectMeta { return metav1.ObjectMeta{Namespace: ns, Name: name} @@ -619,6 +620,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { }, Selector: nil, ClusterIP: "1.2.3.4", + IPFamilyPolicy: &singleStack, SessionAffinity: corev1.ServiceAffinityNone, Type: corev1.ServiceTypeClusterIP, }, diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index f1c493a2259..4b063942e71 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -628,19 +628,14 @@ func (plugin *kubenetNetworkPlugin) getNetworkStatus(id kubecontainer.ContainerI if !ok { return nil } - // sort making v4 first - // TODO: (khenidak) IPv6 beta stage. - // This - forced sort - could be avoided by checking which cidr that an IP belongs - // to, then placing the IP according to cidr index. But before doing that. Check how IP is collected - // across all of kubelet code (against cni and cri). - ips := make([]net.IP, 0) + + if len(iplist) == 0 { + return nil + } + + ips := make([]net.IP, 0, len(iplist)) for _, ip := range iplist { - isV6 := netutils.IsIPv6String(ip) - if !isV6 { - ips = append([]net.IP{net.ParseIP(ip)}, ips...) - } else { - ips = append(ips, net.ParseIP(ip)) - } + ips = append(ips, net.ParseIP(ip)) } return &network.PodNetworkStatus{ diff --git a/pkg/printers/internalversion/printers.go b/pkg/printers/internalversion/printers.go index 6a3bb265f1e..6332746815c 100644 --- a/pkg/printers/internalversion/printers.go +++ b/pkg/printers/internalversion/printers.go @@ -1131,10 +1131,11 @@ func printService(obj *api.Service, options printers.GenerateOptions) ([]metav1. Object: runtime.RawExtension{Object: obj}, } svcType := obj.Spec.Type - internalIP := obj.Spec.ClusterIP - if len(internalIP) == 0 { - internalIP = "" + internalIP := "" + if len(obj.Spec.ClusterIPs) > 0 { + internalIP = obj.Spec.ClusterIPs[0] } + externalIP := getServiceExternalIP(obj, options.Wide) svcPorts := makePortString(obj.Spec.Ports) if len(svcPorts) == 0 { diff --git a/pkg/printers/internalversion/printers_test.go b/pkg/printers/internalversion/printers_test.go index 6d4796a94e4..cf76ac02715 100644 --- a/pkg/printers/internalversion/printers_test.go +++ b/pkg/printers/internalversion/printers_test.go @@ -1074,9 +1074,9 @@ func TestPrintServiceLoadBalancer(t *testing.T) { service: api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "service1"}, Spec: api.ServiceSpec{ - ClusterIP: "1.2.3.4", - Type: "LoadBalancer", - Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}}, + ClusterIPs: []string{"1.2.3.4"}, + Type: "LoadBalancer", + Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}}, }, Status: api.ServiceStatus{ LoadBalancer: api.LoadBalancerStatus{ @@ -1092,9 +1092,9 @@ func TestPrintServiceLoadBalancer(t *testing.T) { service: api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "service2"}, Spec: api.ServiceSpec{ - ClusterIP: "1.3.4.5", - Type: "LoadBalancer", - Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}, {Port: 7777, Protocol: "SCTP"}}, + ClusterIPs: []string{"1.3.4.5"}, + Type: "LoadBalancer", + Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}, {Port: 7777, Protocol: "SCTP"}}, }, }, options: printers.GenerateOptions{}, @@ -1106,9 +1106,9 @@ func TestPrintServiceLoadBalancer(t *testing.T) { service: api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "service3"}, Spec: api.ServiceSpec{ - ClusterIP: "1.4.5.6", - Type: "LoadBalancer", - Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}}, + ClusterIPs: []string{"1.4.5.6"}, + Type: "LoadBalancer", + Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}}, }, Status: api.ServiceStatus{ LoadBalancer: api.LoadBalancerStatus{ @@ -1124,9 +1124,9 @@ func TestPrintServiceLoadBalancer(t *testing.T) { service: api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "service4"}, Spec: api.ServiceSpec{ - ClusterIP: "1.5.6.7", - Type: "LoadBalancer", - Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}}, + ClusterIPs: []string{"1.5.6.7"}, + Type: "LoadBalancer", + Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}}, }, Status: api.ServiceStatus{ LoadBalancer: api.LoadBalancerStatus{ @@ -1142,10 +1142,10 @@ func TestPrintServiceLoadBalancer(t *testing.T) { service: api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "service4"}, Spec: api.ServiceSpec{ - ClusterIP: "1.5.6.7", - Type: "LoadBalancer", - Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}}, - Selector: map[string]string{"foo": "bar"}, + ClusterIPs: []string{"1.5.6.7"}, + Type: "LoadBalancer", + Ports: []api.ServicePort{{Port: 80, Protocol: "TCP"}, {Port: 8090, Protocol: "UDP"}, {Port: 8000, Protocol: "TCP"}}, + Selector: map[string]string{"foo": "bar"}, }, Status: api.ServiceStatus{ LoadBalancer: api.LoadBalancerStatus{ @@ -3083,8 +3083,8 @@ func TestPrintService(t *testing.T) { Port: 2233, }, }, - ClusterIP: "10.9.8.7", - Selector: map[string]string{"foo": "bar"}, // Does NOT get printed. + ClusterIPs: []string{"10.9.8.7"}, + Selector: map[string]string{"foo": "bar"}, // Does NOT get printed. }, }, options: printers.GenerateOptions{}, @@ -3103,8 +3103,8 @@ func TestPrintService(t *testing.T) { Port: 2233, }, }, - ClusterIP: "10.9.8.7", - Selector: map[string]string{"foo": "bar"}, + ClusterIPs: []string{"10.9.8.7"}, + Selector: map[string]string{"foo": "bar"}, }, }, options: printers.GenerateOptions{Wide: true}, @@ -3124,7 +3124,7 @@ func TestPrintService(t *testing.T) { NodePort: 9999, }, }, - ClusterIP: "10.9.8.7", + ClusterIPs: []string{"10.9.8.7"}, }, }, options: printers.GenerateOptions{}, @@ -3143,7 +3143,7 @@ func TestPrintService(t *testing.T) { Port: 8888, }, }, - ClusterIP: "10.9.8.7", + ClusterIPs: []string{"10.9.8.7"}, }, }, options: printers.GenerateOptions{}, @@ -3162,7 +3162,7 @@ func TestPrintService(t *testing.T) { Port: 8888, }, }, - ClusterIP: "10.9.8.7", + ClusterIPs: []string{"10.9.8.7"}, ExternalIPs: singleExternalIP, }, }, @@ -3182,7 +3182,7 @@ func TestPrintService(t *testing.T) { Port: 8888, }, }, - ClusterIP: "10.9.8.7", + ClusterIPs: []string{"10.9.8.7"}, ExternalIPs: singleExternalIP, }, Status: api.ServiceStatus{ @@ -3212,7 +3212,7 @@ func TestPrintService(t *testing.T) { Port: 8888, }, }, - ClusterIP: "10.9.8.7", + ClusterIPs: []string{"10.9.8.7"}, ExternalIPs: mulExternalIP, }, Status: api.ServiceStatus{ @@ -3276,7 +3276,7 @@ func TestPrintServiceList(t *testing.T) { Port: 2233, }, }, - ClusterIP: "10.9.8.7", + ClusterIPs: []string{"10.9.8.7"}, }, }, { @@ -3289,7 +3289,7 @@ func TestPrintServiceList(t *testing.T) { Port: 5566, }, }, - ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, }, }, }, diff --git a/pkg/proxy/apis/config/validation/validation.go b/pkg/proxy/apis/config/validation/validation.go index 019ba44bc5b..e111faf2038 100644 --- a/pkg/proxy/apis/config/validation/validation.go +++ b/pkg/proxy/apis/config/validation/validation.go @@ -75,10 +75,18 @@ func Validate(config *kubeproxyconfig.KubeProxyConfiguration) field.ErrorList { } allErrs = append(allErrs, validateHostPort(config.MetricsBindAddress, newPath.Child("MetricsBindAddress"))...) + dualStackEnabled := effectiveFeatures.Enabled(kubefeatures.IPv6DualStack) + endpointSliceEnabled := effectiveFeatures.Enabled(kubefeatures.EndpointSlice) + + // dual stack has strong dependency on endpoint slice since + // endpoint slice controller is the only capabable of producing + // slices for *all* clusterIPs + if dualStackEnabled && !endpointSliceEnabled { + allErrs = append(allErrs, field.Invalid(newPath.Child("FeatureGates"), config.FeatureGates, "EndpointSlice feature flag must be turned on when turning on DualStack")) + } + if config.ClusterCIDR != "" { cidrs := strings.Split(config.ClusterCIDR, ",") - dualStackEnabled := effectiveFeatures.Enabled(kubefeatures.IPv6DualStack) - switch { // if DualStack only valid one cidr or two cidrs with one of each IP family case dualStackEnabled && len(cidrs) > 2: diff --git a/pkg/proxy/apis/config/validation/validation_test.go b/pkg/proxy/apis/config/validation/validation_test.go index 35bf9d77e0b..c01252c27e7 100644 --- a/pkg/proxy/apis/config/validation/validation_test.go +++ b/pkg/proxy/apis/config/validation/validation_test.go @@ -124,7 +124,7 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { BindAddress: "10.10.12.11", HealthzBindAddress: "0.0.0.0:12345", MetricsBindAddress: "127.0.0.1:10249", - FeatureGates: map[string]bool{"IPv6DualStack": true}, + FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": true}, ClusterCIDR: "192.168.59.0/24", UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, @@ -290,7 +290,7 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { HealthzBindAddress: "0.0.0.0:12345", MetricsBindAddress: "127.0.0.1:10249", // DualStack ClusterCIDR without feature flag enabled - FeatureGates: map[string]bool{"IPv6DualStack": false}, + FeatureGates: map[string]bool{"IPv6DualStack": false, "EndpointSlice": false}, ClusterCIDR: "192.168.59.0/24,fd00:192:168::/64", UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, @@ -308,13 +308,38 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { }, msg: "only one CIDR allowed (e.g. 10.100.0.0/16 or fde4:8dba:82e1::/48)", }, + { + config: kubeproxyconfig.KubeProxyConfiguration{ + BindAddress: "10.10.12.11", + HealthzBindAddress: "0.0.0.0:12345", + MetricsBindAddress: "127.0.0.1:10249", + // DualStack ClusterCIDR with feature flag enabled but EndpointSlice is not enabled + FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": false}, + ClusterCIDR: "192.168.59.0/24,fd00:192:168::/64", + UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, + ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ + MaxPerCore: pointer.Int32Ptr(1), + Min: pointer.Int32Ptr(1), + TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, + TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, + }, + }, + msg: "EndpointSlice feature flag must be turned on", + }, + { config: kubeproxyconfig.KubeProxyConfiguration{ BindAddress: "10.10.12.11", HealthzBindAddress: "0.0.0.0:12345", MetricsBindAddress: "127.0.0.1:10249", // DualStack with multiple CIDRs but only one IP family - FeatureGates: map[string]bool{"IPv6DualStack": true}, + FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": true}, ClusterCIDR: "192.168.59.0/24,10.0.0.0/16", UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, @@ -338,7 +363,7 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { HealthzBindAddress: "0.0.0.0:12345", MetricsBindAddress: "127.0.0.1:10249", // DualStack with an invalid subnet - FeatureGates: map[string]bool{"IPv6DualStack": true}, + FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": true}, ClusterCIDR: "192.168.59.0/24,fd00:192:168::/64,a.b.c.d/f", UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, @@ -361,7 +386,7 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { BindAddress: "10.10.12.11", HealthzBindAddress: "0.0.0.0:12345", MetricsBindAddress: "127.0.0.1:10249", - FeatureGates: map[string]bool{"IPv6DualStack": true}, + FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": true}, ClusterCIDR: "192.168.59.0/24,fd00:192:168::/64,10.0.0.0/16", UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, diff --git a/pkg/proxy/endpoints.go b/pkg/proxy/endpoints.go index 234cf4f2656..fdbe1315be2 100644 --- a/pkg/proxy/endpoints.go +++ b/pkg/proxy/endpoints.go @@ -112,27 +112,27 @@ type EndpointChangeTracker struct { processEndpointsMapChange processEndpointsMapChangeFunc // endpointSliceCache holds a simplified version of endpoint slices. endpointSliceCache *EndpointSliceCache - // isIPv6Mode indicates if change tracker is under IPv6/IPv4 mode. Nil means not applicable. - isIPv6Mode *bool - recorder record.EventRecorder + // ipfamily identify the ip family on which the tracker is operating on + ipFamily v1.IPFamily + recorder record.EventRecorder // Map from the Endpoints namespaced-name to the times of the triggers that caused the endpoints // object to change. Used to calculate the network-programming-latency. lastChangeTriggerTimes map[types.NamespacedName][]time.Time } // NewEndpointChangeTracker initializes an EndpointsChangeMap -func NewEndpointChangeTracker(hostname string, makeEndpointInfo makeEndpointFunc, isIPv6Mode *bool, recorder record.EventRecorder, endpointSlicesEnabled bool, processEndpointsMapChange processEndpointsMapChangeFunc) *EndpointChangeTracker { +func NewEndpointChangeTracker(hostname string, makeEndpointInfo makeEndpointFunc, ipFamily v1.IPFamily, recorder record.EventRecorder, endpointSlicesEnabled bool, processEndpointsMapChange processEndpointsMapChangeFunc) *EndpointChangeTracker { ect := &EndpointChangeTracker{ hostname: hostname, items: make(map[types.NamespacedName]*endpointsChange), makeEndpointInfo: makeEndpointInfo, - isIPv6Mode: isIPv6Mode, + ipFamily: ipFamily, recorder: recorder, lastChangeTriggerTimes: make(map[types.NamespacedName][]time.Time), processEndpointsMapChange: processEndpointsMapChange, } if endpointSlicesEnabled { - ect.endpointSliceCache = NewEndpointSliceCache(hostname, isIPv6Mode, recorder, makeEndpointInfo) + ect.endpointSliceCache = NewEndpointSliceCache(hostname, ipFamily, recorder, makeEndpointInfo) } return ect } @@ -374,14 +374,16 @@ func (ect *EndpointChangeTracker) endpointsToEndpointsMap(endpoints *v1.Endpoint klog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name) continue } + // Filter out the incorrect IP version case. // Any endpoint port that contains incorrect IP version will be ignored. - if ect.isIPv6Mode != nil && utilnet.IsIPv6String(addr.IP) != *ect.isIPv6Mode { + if (ect.ipFamily == v1.IPv6Protocol) != utilnet.IsIPv6String(addr.IP) { // Emit event on the corresponding service which had a different // IP version than the endpoint. utilproxy.LogAndEmitIncorrectIPVersionEvent(ect.recorder, "endpoints", addr.IP, endpoints.Namespace, endpoints.Name, "") continue } + isLocal := addr.NodeName != nil && *addr.NodeName == ect.hostname baseEndpointInfo := newBaseEndpointInfo(addr.IP, int(port.Port), isLocal, nil) if ect.makeEndpointInfo != nil { diff --git a/pkg/proxy/endpoints_test.go b/pkg/proxy/endpoints_test.go index 820e5bfccfe..173e3182ebb 100644 --- a/pkg/proxy/endpoints_test.go +++ b/pkg/proxy/endpoints_test.go @@ -135,24 +135,24 @@ func makeTestEndpoints(namespace, name string, eptFunc func(*v1.Endpoints)) *v1. // This is a coarse test, but it offers some modicum of confidence as the code is evolved. func TestEndpointsToEndpointsMap(t *testing.T) { - epTracker := NewEndpointChangeTracker("test-hostname", nil, nil, nil, false, nil) - - trueVal := true - falseVal := false - testCases := []struct { desc string newEndpoints *v1.Endpoints expected map[ServicePortName][]*BaseEndpointInfo isIPv6Mode *bool + ipFamily v1.IPFamily }{ { - desc: "nothing", + desc: "nothing", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) {}), expected: map[ServicePortName][]*BaseEndpointInfo{}, }, { - desc: "no changes, unnamed port", + desc: "no changes, unnamed port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -174,7 +174,9 @@ func TestEndpointsToEndpointsMap(t *testing.T) { }, }, { - desc: "no changes, named port", + desc: "no changes, named port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -196,7 +198,9 @@ func TestEndpointsToEndpointsMap(t *testing.T) { }, }, { - desc: "new port", + desc: "new port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -217,12 +221,16 @@ func TestEndpointsToEndpointsMap(t *testing.T) { }, }, { - desc: "remove port", + desc: "remove port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) {}), expected: map[ServicePortName][]*BaseEndpointInfo{}, }, { - desc: "new IP and port", + desc: "new IP and port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -255,7 +263,9 @@ func TestEndpointsToEndpointsMap(t *testing.T) { }, }, { - desc: "remove IP and port", + desc: "remove IP and port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -277,7 +287,9 @@ func TestEndpointsToEndpointsMap(t *testing.T) { }, }, { - desc: "rename port", + desc: "rename port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -299,7 +311,9 @@ func TestEndpointsToEndpointsMap(t *testing.T) { }, }, { - desc: "renumber port", + desc: "renumber port", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -321,7 +335,9 @@ func TestEndpointsToEndpointsMap(t *testing.T) { }, }, { - desc: "should omit IPv6 address in IPv4 mode", + desc: "should omit IPv6 address in IPv4 mode", + ipFamily: v1.IPv4Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -350,10 +366,11 @@ func TestEndpointsToEndpointsMap(t *testing.T) { {Endpoint: "1.1.1.1:22", IsLocal: false}, }, }, - isIPv6Mode: &falseVal, }, { - desc: "should omit IPv4 address in IPv6 mode", + desc: "should omit IPv4 address in IPv6 mode", + ipFamily: v1.IPv6Protocol, + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{ { @@ -382,30 +399,33 @@ func TestEndpointsToEndpointsMap(t *testing.T) { {Endpoint: "[2001:db8:85a3:0:0:8a2e:370:7334]:22", IsLocal: false}, }, }, - isIPv6Mode: &trueVal, }, } for _, tc := range testCases { - epTracker.isIPv6Mode = tc.isIPv6Mode - // outputs - newEndpoints := epTracker.endpointsToEndpointsMap(tc.newEndpoints) + t.Run(tc.desc, func(t *testing.T) { - if len(newEndpoints) != len(tc.expected) { - t.Errorf("[%s] expected %d new, got %d: %v", tc.desc, len(tc.expected), len(newEndpoints), spew.Sdump(newEndpoints)) - } - for x := range tc.expected { - if len(newEndpoints[x]) != len(tc.expected[x]) { - t.Errorf("[%s] expected %d endpoints for %v, got %d", tc.desc, len(tc.expected[x]), x, len(newEndpoints[x])) - } else { - for i := range newEndpoints[x] { - ep := newEndpoints[x][i].(*BaseEndpointInfo) - if !(reflect.DeepEqual(*ep, *(tc.expected[x][i]))) { - t.Errorf("[%s] expected new[%v][%d] to be %v, got %v", tc.desc, x, i, tc.expected[x][i], *ep) + epTracker := NewEndpointChangeTracker("test-hostname", nil, tc.ipFamily, nil, false, nil) + + // outputs + newEndpoints := epTracker.endpointsToEndpointsMap(tc.newEndpoints) + + if len(newEndpoints) != len(tc.expected) { + t.Fatalf("[%s] expected %d new, got %d: %v", tc.desc, len(tc.expected), len(newEndpoints), spew.Sdump(newEndpoints)) + } + for x := range tc.expected { + if len(newEndpoints[x]) != len(tc.expected[x]) { + t.Fatalf("[%s] expected %d endpoints for %v, got %d", tc.desc, len(tc.expected[x]), x, len(newEndpoints[x])) + } else { + for i := range newEndpoints[x] { + ep := newEndpoints[x][i].(*BaseEndpointInfo) + if !(reflect.DeepEqual(*ep, *(tc.expected[x][i]))) { + t.Fatalf("[%s] expected new[%v][%d] to be %v, got %v", tc.desc, x, i, tc.expected[x][i], *ep) + } } } } - } + }) } } @@ -1249,7 +1269,7 @@ func TestUpdateEndpointsMap(t *testing.T) { for tci, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fp := newFakeProxier() + fp := newFakeProxier(v1.IPv4Protocol) fp.hostname = nodeName // First check that after adding all previous versions of endpoints, @@ -1424,7 +1444,7 @@ func TestLastChangeTriggerTime(t *testing.T) { } for _, tc := range testCases { - fp := newFakeProxier() + fp := newFakeProxier(v1.IPv4Protocol) tc.scenario(fp) @@ -1454,7 +1474,7 @@ func TestEndpointSliceUpdate(t *testing.T) { // test starting from an empty state "add a simple slice that doesn't already exist": { startingSlices: []*discovery.EndpointSlice{}, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: false, @@ -1477,7 +1497,7 @@ func TestEndpointSliceUpdate(t *testing.T) { startingSlices: []*discovery.EndpointSlice{ generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: false, @@ -1489,7 +1509,7 @@ func TestEndpointSliceUpdate(t *testing.T) { startingSlices: []*discovery.EndpointSlice{ generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: fqdnSlice, paramRemoveSlice: false, @@ -1502,7 +1522,7 @@ func TestEndpointSliceUpdate(t *testing.T) { generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), generateEndpointSlice("svc1", "ns1", 2, 2, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 1, 5, 999, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: false, @@ -1534,7 +1554,7 @@ func TestEndpointSliceUpdate(t *testing.T) { generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), generateEndpointSlice("svc1", "ns1", 2, 2, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSliceWithOffset("svc1", "ns1", 3, 1, 5, 999, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80)}), paramRemoveSlice: false, @@ -1564,7 +1584,7 @@ func TestEndpointSliceUpdate(t *testing.T) { generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), generateEndpointSlice("svc1", "ns1", 2, 2, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 1, 5, 999, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: true, @@ -1586,7 +1606,7 @@ func TestEndpointSliceUpdate(t *testing.T) { generateEndpointSlice("svc1", "ns1", 1, 5, 999, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), generateEndpointSlice("svc1", "ns1", 2, 2, 999, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 3, 5, 999, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: true, @@ -1598,7 +1618,7 @@ func TestEndpointSliceUpdate(t *testing.T) { startingSlices: []*discovery.EndpointSlice{ generateEndpointSlice("svc1", "ns1", 1, 3, 999, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 1, 3, 1, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: false, @@ -1610,7 +1630,7 @@ func TestEndpointSliceUpdate(t *testing.T) { startingSlices: []*discovery.EndpointSlice{ generateEndpointSlice("svc1", "ns1", 1, 2, 1, []string{"host1", "host2"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 1, 2, 999, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: false, @@ -1632,7 +1652,7 @@ func TestEndpointSliceUpdate(t *testing.T) { generateEndpointSlice("svc1", "ns1", 1, 3, 2, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), generateEndpointSlice("svc1", "ns1", 2, 2, 2, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), }, - endpointChangeTracker: NewEndpointChangeTracker("host1", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("host1", nil, v1.IPv4Protocol, nil, true, nil), namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"}, paramEndpointSlice: generateEndpointSlice("svc1", "ns1", 1, 3, 3, []string{"host1"}, []*int32{utilpointer.Int32Ptr(80), utilpointer.Int32Ptr(443)}), paramRemoveSlice: false, @@ -1691,20 +1711,20 @@ func TestCheckoutChanges(t *testing.T) { pendingSlices []*discovery.EndpointSlice }{ "empty slices": { - endpointChangeTracker: NewEndpointChangeTracker("", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("", nil, v1.IPv4Protocol, nil, true, nil), expectedChanges: []*endpointsChange{}, useEndpointSlices: true, appliedSlices: []*discovery.EndpointSlice{}, pendingSlices: []*discovery.EndpointSlice{}, }, "without slices, empty items": { - endpointChangeTracker: NewEndpointChangeTracker("", nil, nil, nil, false, nil), + endpointChangeTracker: NewEndpointChangeTracker("", nil, v1.IPv4Protocol, nil, false, nil), expectedChanges: []*endpointsChange{}, items: map[types.NamespacedName]*endpointsChange{}, useEndpointSlices: false, }, "without slices, simple items": { - endpointChangeTracker: NewEndpointChangeTracker("", nil, nil, nil, false, nil), + endpointChangeTracker: NewEndpointChangeTracker("", nil, v1.IPv4Protocol, nil, false, nil), expectedChanges: []*endpointsChange{{ previous: EndpointsMap{ svcPortName0: []Endpoint{newTestEp("10.0.1.1:80", ""), newTestEp("10.0.1.2:80", "")}, @@ -1728,7 +1748,7 @@ func TestCheckoutChanges(t *testing.T) { useEndpointSlices: false, }, "adding initial slice": { - endpointChangeTracker: NewEndpointChangeTracker("", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("", nil, v1.IPv4Protocol, nil, true, nil), expectedChanges: []*endpointsChange{{ previous: EndpointsMap{}, current: EndpointsMap{ @@ -1742,7 +1762,7 @@ func TestCheckoutChanges(t *testing.T) { }, }, "removing port in update": { - endpointChangeTracker: NewEndpointChangeTracker("", nil, nil, nil, true, nil), + endpointChangeTracker: NewEndpointChangeTracker("", nil, v1.IPv4Protocol, nil, true, nil), expectedChanges: []*endpointsChange{{ previous: EndpointsMap{ svcPortName0: []Endpoint{newTestEp("10.0.1.1:80", "host1"), newTestEp("10.0.1.2:80", "host1")}, @@ -1802,24 +1822,24 @@ func TestCheckoutChanges(t *testing.T) { func compareEndpointsMapsStr(t *testing.T, newMap EndpointsMap, expected map[ServicePortName][]*BaseEndpointInfo) { t.Helper() if len(newMap) != len(expected) { - t.Errorf("expected %d results, got %d: %v", len(expected), len(newMap), newMap) + t.Fatalf("expected %d results, got %d: %v", len(expected), len(newMap), newMap) } endpointEqual := func(a, b *BaseEndpointInfo) bool { return a.Endpoint == b.Endpoint && a.IsLocal == b.IsLocal } for x := range expected { if len(newMap[x]) != len(expected[x]) { - t.Errorf("expected %d endpoints for %v, got %d", len(expected[x]), x, len(newMap[x])) t.Logf("Endpoints %+v", newMap[x]) + t.Fatalf("expected %d endpoints for %v, got %d", len(expected[x]), x, len(newMap[x])) } else { for i := range expected[x] { newEp, ok := newMap[x][i].(*BaseEndpointInfo) if !ok { - t.Errorf("Failed to cast endpointsInfo") + t.Fatalf("Failed to cast endpointsInfo") continue } if !endpointEqual(newEp, expected[x][i]) { - t.Errorf("expected new[%v][%d] to be %v, got %v (IsLocal expected %v, got %v)", x, i, expected[x][i], newEp, expected[x][i].IsLocal, newEp.IsLocal) + t.Fatalf("expected new[%v][%d] to be %v, got %v (IsLocal expected %v, got %v)", x, i, expected[x][i], newEp, expected[x][i].IsLocal, newEp.IsLocal) } } } diff --git a/pkg/proxy/endpointslicecache.go b/pkg/proxy/endpointslicecache.go index b4adee93c42..b049481a372 100644 --- a/pkg/proxy/endpointslicecache.go +++ b/pkg/proxy/endpointslicecache.go @@ -47,7 +47,7 @@ type EndpointSliceCache struct { makeEndpointInfo makeEndpointFunc hostname string - isIPv6Mode *bool + ipFamily v1.IPFamily recorder record.EventRecorder } @@ -84,14 +84,14 @@ type endpointInfo struct { type spToEndpointMap map[ServicePortName]map[string]Endpoint // NewEndpointSliceCache initializes an EndpointSliceCache. -func NewEndpointSliceCache(hostname string, isIPv6Mode *bool, recorder record.EventRecorder, makeEndpointInfo makeEndpointFunc) *EndpointSliceCache { +func NewEndpointSliceCache(hostname string, ipFamily v1.IPFamily, recorder record.EventRecorder, makeEndpointInfo makeEndpointFunc) *EndpointSliceCache { if makeEndpointInfo == nil { makeEndpointInfo = standardEndpointInfo } return &EndpointSliceCache{ trackerByServiceMap: map[types.NamespacedName]*endpointSliceTracker{}, hostname: hostname, - isIPv6Mode: isIPv6Mode, + ipFamily: ipFamily, makeEndpointInfo: makeEndpointInfo, recorder: recorder, } @@ -248,7 +248,7 @@ func (cache *EndpointSliceCache) addEndpointsByIP(serviceNN types.NamespacedName // Filter out the incorrect IP version case. Any endpoint port that // contains incorrect IP version will be ignored. - if cache.isIPv6Mode != nil && utilnet.IsIPv6String(endpoint.Addresses[0]) != *cache.isIPv6Mode { + if (cache.ipFamily == v1.IPv6Protocol) != utilnet.IsIPv6String(endpoint.Addresses[0]) { // Emit event on the corresponding service which had a different IP // version than the endpoint. utilproxy.LogAndEmitIncorrectIPVersionEvent(cache.recorder, "endpointslice", endpoint.Addresses[0], serviceNN.Namespace, serviceNN.Name, "") diff --git a/pkg/proxy/endpointslicecache_test.go b/pkg/proxy/endpointslicecache_test.go index 1e45c7a7bb1..ebe4afa7e1d 100644 --- a/pkg/proxy/endpointslicecache_test.go +++ b/pkg/proxy/endpointslicecache_test.go @@ -151,7 +151,7 @@ func TestEndpointsMapFromESC(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - esCache := NewEndpointSliceCache(tc.hostname, nil, nil, nil) + esCache := NewEndpointSliceCache(tc.hostname, v1.IPv4Protocol, nil, nil) cmc := newCacheMutationCheck(tc.endpointSlices) for _, endpointSlice := range tc.endpointSlices { @@ -189,7 +189,7 @@ func TestEndpointInfoByServicePort(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - esCache := NewEndpointSliceCache(tc.hostname, nil, nil, nil) + esCache := NewEndpointSliceCache(tc.hostname, v1.IPv4Protocol, nil, nil) for _, endpointSlice := range tc.endpointSlices { esCache.updatePending(endpointSlice, false) @@ -225,7 +225,7 @@ func TestEsInfoChanged(t *testing.T) { expectChanged bool }{ "identical slices, ports only": { - cache: NewEndpointSliceCache("", nil, nil, nil), + cache: NewEndpointSliceCache("", v1.IPv4Protocol, nil, nil), initialSlice: &discovery.EndpointSlice{ ObjectMeta: objMeta, Ports: []discovery.EndpointPort{port80}, @@ -237,7 +237,7 @@ func TestEsInfoChanged(t *testing.T) { expectChanged: false, }, "identical slices, ports out of order": { - cache: NewEndpointSliceCache("", nil, nil, nil), + cache: NewEndpointSliceCache("", v1.IPv4Protocol, nil, nil), initialSlice: &discovery.EndpointSlice{ ObjectMeta: objMeta, Ports: []discovery.EndpointPort{port443, port80}, @@ -249,7 +249,7 @@ func TestEsInfoChanged(t *testing.T) { expectChanged: false, }, "port removed": { - cache: NewEndpointSliceCache("", nil, nil, nil), + cache: NewEndpointSliceCache("", v1.IPv4Protocol, nil, nil), initialSlice: &discovery.EndpointSlice{ ObjectMeta: objMeta, Ports: []discovery.EndpointPort{port443, port80}, @@ -261,7 +261,7 @@ func TestEsInfoChanged(t *testing.T) { expectChanged: true, }, "port added": { - cache: NewEndpointSliceCache("", nil, nil, nil), + cache: NewEndpointSliceCache("", v1.IPv4Protocol, nil, nil), initialSlice: &discovery.EndpointSlice{ ObjectMeta: objMeta, Ports: []discovery.EndpointPort{port443}, @@ -273,7 +273,7 @@ func TestEsInfoChanged(t *testing.T) { expectChanged: true, }, "identical with endpoints": { - cache: NewEndpointSliceCache("", nil, nil, nil), + cache: NewEndpointSliceCache("", v1.IPv4Protocol, nil, nil), initialSlice: &discovery.EndpointSlice{ ObjectMeta: objMeta, Ports: []discovery.EndpointPort{port443}, @@ -287,7 +287,7 @@ func TestEsInfoChanged(t *testing.T) { expectChanged: false, }, "identical with endpoints out of order": { - cache: NewEndpointSliceCache("", nil, nil, nil), + cache: NewEndpointSliceCache("", v1.IPv4Protocol, nil, nil), initialSlice: &discovery.EndpointSlice{ ObjectMeta: objMeta, Ports: []discovery.EndpointPort{port443}, @@ -301,7 +301,7 @@ func TestEsInfoChanged(t *testing.T) { expectChanged: false, }, "identical with endpoint added": { - cache: NewEndpointSliceCache("", nil, nil, nil), + cache: NewEndpointSliceCache("", v1.IPv4Protocol, nil, nil), initialSlice: &discovery.EndpointSlice{ ObjectMeta: objMeta, Ports: []discovery.EndpointPort{port443}, diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 61cb25a4851..a3946d83c4c 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -289,18 +289,23 @@ func NewProxier(ipt utiliptables.Interface, serviceHealthServer := healthcheck.NewServiceHealthServer(hostname, recorder) - isIPv6 := ipt.IsIPv6() + ipFamily := v1.IPv4Protocol + if ipt.IsIPv6() { + ipFamily = v1.IPv6Protocol + } + var incorrectAddresses []string - nodePortAddresses, incorrectAddresses = utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, isIPv6) + nodePortAddresses, incorrectAddresses = utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, ipFamily) if len(incorrectAddresses) > 0 { klog.Warning("NodePortAddresses of wrong family; ", incorrectAddresses) } + proxier := &Proxier{ portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, &isIPv6, recorder, nil), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, ipFamily, recorder, nil), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(hostname, newEndpointInfo, &isIPv6, recorder, endpointSlicesEnabled, nil), + endpointsChanges: proxy.NewEndpointChangeTracker(hostname, newEndpointInfo, ipFamily, recorder, endpointSlicesEnabled, nil), syncPeriod: syncPeriod, iptables: ipt, masqueradeAll: masqueradeAll, @@ -362,7 +367,7 @@ func NewDualStackProxier( nodePortAddresses []string, ) (proxy.Provider, error) { // Create an ipv4 instance of the single-stack proxier - nodePortAddresses4, nodePortAddresses6 := utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, false) + nodePortAddresses4, nodePortAddresses6 := utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, v1.IPv4Protocol) ipv4Proxier, err := NewProxier(ipt[0], sysctl, exec, syncPeriod, minSyncPeriod, masqueradeAll, masqueradeBit, localDetectors[0], hostname, nodeIP[0], recorder, healthzServer, nodePortAddresses4) @@ -376,8 +381,7 @@ func NewDualStackProxier( if err != nil { return nil, fmt.Errorf("unable to create ipv6 proxier: %v", err) } - - return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil // TODO move meta-proxier to mode-neutral package + return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil } type iptablesJumpChain struct { diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 8877549ff9f..0cae52809dd 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -150,8 +150,7 @@ func TestGetChainLinesMultipleTables(t *testing.T) { } checkAllLines(t, utiliptables.TableNAT, []byte(iptablesSave), expected) } - -func TestDeleteEndpointConnections(t *testing.T) { +func TestDeleteEndpointConnectionsIPv4(t *testing.T) { const ( UDP = v1.ProtocolUDP TCP = v1.ProtocolTCP @@ -175,21 +174,24 @@ func TestDeleteEndpointConnections(t *testing.T) { svcPort: 80, protocol: UDP, endpoint: "10.240.0.3:80", - }, { + }, + { description: "V4 TCP", svcName: "v4-tcp", svcIP: "10.96.2.2", svcPort: 80, protocol: TCP, endpoint: "10.240.0.4:80", - }, { + }, + { description: "V4 SCTP", svcName: "v4-sctp", svcIP: "10.96.3.3", svcPort: 80, protocol: SCTP, endpoint: "10.240.0.5:80", - }, { + }, + { description: "V4 UDP, nothing to delete, benign error", svcName: "v4-udp-nothing-to-delete", svcIP: "10.96.1.1", @@ -197,7 +199,8 @@ func TestDeleteEndpointConnections(t *testing.T) { protocol: UDP, endpoint: "10.240.0.3:80", simulatedErr: conntrack.NoConnectionToDelete, - }, { + }, + { description: "V4 UDP, unexpected error, should be glogged", svcName: "v4-udp-simulated-error", svcIP: "10.96.1.1", @@ -205,27 +208,6 @@ func TestDeleteEndpointConnections(t *testing.T) { protocol: UDP, endpoint: "10.240.0.3:80", simulatedErr: "simulated error", - }, { - description: "V6 UDP", - svcName: "v6-udp", - svcIP: "fd00:1234::20", - svcPort: 80, - protocol: UDP, - endpoint: "[2001:db8::2]:80", - }, { - description: "V6 TCP", - svcName: "v6-tcp", - svcIP: "fd00:1234::30", - svcPort: 80, - protocol: TCP, - endpoint: "[2001:db8::3]:80", - }, { - description: "V6 SCTP", - svcName: "v6-sctp", - svcIP: "fd00:1234::40", - svcPort: 80, - protocol: SCTP, - endpoint: "[2001:db8::4]:80", }, } @@ -329,6 +311,149 @@ func TestDeleteEndpointConnections(t *testing.T) { } } +func TestDeleteEndpointConnectionsIPv6(t *testing.T) { + const ( + UDP = v1.ProtocolUDP + TCP = v1.ProtocolTCP + SCTP = v1.ProtocolSCTP + ) + + testCases := []struct { + description string + svcName string + svcIP string + svcPort int32 + protocol v1.Protocol + endpoint string // IP:port endpoint + epSvcPair proxy.ServiceEndpoint // Will be generated by test + simulatedErr string + }{ + { + description: "V6 UDP", + svcName: "v6-udp", + svcIP: "fd00:1234::20", + svcPort: 80, + protocol: UDP, + endpoint: "[2001:db8::2]:80", + }, + { + description: "V6 TCP", + svcName: "v6-tcp", + svcIP: "fd00:1234::30", + svcPort: 80, + protocol: TCP, + endpoint: "[2001:db8::3]:80", + }, + { + description: "V6 SCTP", + svcName: "v6-sctp", + svcIP: "fd00:1234::40", + svcPort: 80, + protocol: SCTP, + endpoint: "[2001:db8::4]:80", + }, + } + + // Create a fake executor for the conntrack utility. This should only be + // invoked for UDP and SCTP connections, since no conntrack cleanup is needed for TCP + fcmd := fakeexec.FakeCmd{} + fexec := fakeexec.FakeExec{ + LookPathFunc: func(cmd string) (string, error) { return cmd, nil }, + } + execFunc := func(cmd string, args ...string) exec.Cmd { + return fakeexec.InitFakeCmd(&fcmd, cmd, args...) + } + for _, tc := range testCases { + if conntrack.IsClearConntrackNeeded(tc.protocol) { + var cmdOutput string + var simErr error + if tc.simulatedErr == "" { + cmdOutput = "1 flow entries have been deleted" + } else { + simErr = fmt.Errorf(tc.simulatedErr) + } + cmdFunc := func() ([]byte, []byte, error) { return []byte(cmdOutput), nil, simErr } + fcmd.CombinedOutputScript = append(fcmd.CombinedOutputScript, cmdFunc) + fexec.CommandScript = append(fexec.CommandScript, execFunc) + } + } + + ipt := iptablestest.NewIPv6Fake() + fp := NewFakeProxier(ipt, false) + fp.exec = &fexec + + for _, tc := range testCases { + makeServiceMap(fp, + makeTestService("ns1", tc.svcName, func(svc *v1.Service) { + svc.Spec.ClusterIP = tc.svcIP + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: tc.svcPort, + Protocol: tc.protocol, + }} + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + }), + ) + + proxy.UpdateServiceMap(fp.serviceMap, fp.serviceChanges) + } + + // Run the test cases + for _, tc := range testCases { + priorExecs := fexec.CommandCalls + priorGlogErrs := klog.Stats.Error.Lines() + + svc := proxy.ServicePortName{ + NamespacedName: types.NamespacedName{Namespace: "ns1", Name: tc.svcName}, + Port: "p80", + Protocol: tc.protocol, + } + input := []proxy.ServiceEndpoint{ + { + Endpoint: tc.endpoint, + ServicePortName: svc, + }, + } + + fp.deleteEndpointConnections(input) + + // For UDP and SCTP connections, check the executed conntrack command + var expExecs int + if conntrack.IsClearConntrackNeeded(tc.protocol) { + isIPv6 := func(ip string) bool { + netIP := net.ParseIP(ip) + return netIP.To4() == nil + } + endpointIP := utilproxy.IPPart(tc.endpoint) + expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s --dst-nat %s -p %s", tc.svcIP, endpointIP, strings.ToLower(string((tc.protocol)))) + if isIPv6(endpointIP) { + expectCommand += " -f ipv6" + } + actualCommand := strings.Join(fcmd.CombinedOutputLog[fexec.CommandCalls-1], " ") + if actualCommand != expectCommand { + t.Errorf("%s: Expected command: %s, but executed %s", tc.description, expectCommand, actualCommand) + } + expExecs = 1 + } + + // Check the number of times conntrack was executed + execs := fexec.CommandCalls - priorExecs + if execs != expExecs { + t.Errorf("%s: Expected conntrack to be executed %d times, but got %d", tc.description, expExecs, execs) + } + + // Check the number of new glog errors + var expGlogErrs int64 + if tc.simulatedErr != "" && tc.simulatedErr != conntrack.NoConnectionToDelete { + expGlogErrs = 1 + } + glogErrs := klog.Stats.Error.Lines() - priorGlogErrs + if glogErrs != expGlogErrs { + t.Errorf("%s: Expected %d glogged errors, but got %d", tc.description, expGlogErrs, glogErrs) + } + } +} + // fakePortOpener implements portOpener. type fakePortOpener struct { openPorts []*utilproxy.LocalPort @@ -346,13 +471,17 @@ const testHostname = "test-hostname" func NewFakeProxier(ipt utiliptables.Interface, endpointSlicesEnabled bool) *Proxier { // TODO: Call NewProxier after refactoring out the goroutine // invocation into a Run() method. + ipfamily := v1.IPv4Protocol + if ipt.IsIPv6() { + ipfamily = v1.IPv6Protocol + } detectLocal, _ := proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", ipt) p := &Proxier{ exec: &fakeexec.FakeExec{}, serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, nil, nil, nil), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, ipfamily, nil, nil), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(testHostname, newEndpointInfo, nil, nil, endpointSlicesEnabled, nil), + endpointsChanges: proxy.NewEndpointChangeTracker(testHostname, newEndpointInfo, ipfamily, nil, endpointSlicesEnabled, nil), iptables: ipt, masqueradeMark: "0x4000", localDetector: detectLocal, diff --git a/pkg/proxy/ipvs/BUILD b/pkg/proxy/ipvs/BUILD index 52544641d62..f10643ece38 100644 --- a/pkg/proxy/ipvs/BUILD +++ b/pkg/proxy/ipvs/BUILD @@ -37,6 +37,7 @@ go_test( "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 5f037963069..a34b181640d 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -202,6 +202,8 @@ const sysctlArpAnnounce = "net/ipv4/conf/all/arp_announce" // Proxier is an ipvs based proxy for connections between a localhost:lport // and services that provide the actual backends. type Proxier struct { + // the ipfamily on which this proxy is operating on. + ipFamily v1.IPFamily // endpointsChanges and serviceChanges contains all changes to endpoints and // services that happened since last syncProxyRules call. For a single object, // changes are accumulated, i.e. previous is state from before all of them, @@ -432,9 +434,12 @@ func NewProxier(ipt utiliptables.Interface, masqueradeValue := 1 << uint(masqueradeBit) masqueradeMark := fmt.Sprintf("%#08x", masqueradeValue) - isIPv6 := utilnet.IsIPv6(nodeIP) + ipFamily := v1.IPv4Protocol + if ipt.IsIPv6() { + ipFamily = v1.IPv6Protocol + } - klog.V(2).Infof("nodeIP: %v, isIPv6: %v", nodeIP, isIPv6) + klog.V(2).Infof("nodeIP: %v, family: %v", nodeIP, ipFamily) if len(scheduler) == 0 { klog.Warningf("IPVS scheduler not specified, use %s by default", DefaultScheduler) @@ -446,16 +451,17 @@ func NewProxier(ipt utiliptables.Interface, endpointSlicesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying) var incorrectAddresses []string - nodePortAddresses, incorrectAddresses = utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, isIPv6) + nodePortAddresses, incorrectAddresses = utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, ipFamily) if len(incorrectAddresses) > 0 { klog.Warning("NodePortAddresses of wrong family; ", incorrectAddresses) } proxier := &Proxier{ + ipFamily: ipFamily, portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, &isIPv6, recorder, nil), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, ipFamily, recorder, nil), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(hostname, nil, &isIPv6, recorder, endpointSlicesEnabled, nil), + endpointsChanges: proxy.NewEndpointChangeTracker(hostname, nil, ipFamily, recorder, endpointSlicesEnabled, nil), syncPeriod: syncPeriod, minSyncPeriod: minSyncPeriod, excludeCIDRs: parseExcludedCIDRs(excludeCIDRs), @@ -472,14 +478,14 @@ func NewProxier(ipt utiliptables.Interface, healthzServer: healthzServer, ipvs: ipvs, ipvsScheduler: scheduler, - ipGetter: &realIPGetter{nl: NewNetLinkHandle(isIPv6)}, + ipGetter: &realIPGetter{nl: NewNetLinkHandle(ipFamily == v1.IPv6Protocol)}, iptablesData: bytes.NewBuffer(nil), filterChainsData: bytes.NewBuffer(nil), natChains: bytes.NewBuffer(nil), natRules: bytes.NewBuffer(nil), filterChains: bytes.NewBuffer(nil), filterRules: bytes.NewBuffer(nil), - netlinkHandle: NewNetLinkHandle(isIPv6), + netlinkHandle: NewNetLinkHandle(ipFamily == v1.IPv6Protocol), ipset: ipset, nodePortAddresses: nodePortAddresses, networkInterfacer: utilproxy.RealNetwork{}, @@ -488,7 +494,7 @@ func NewProxier(ipt utiliptables.Interface, // initialize ipsetList with all sets we needed proxier.ipsetList = make(map[string]*IPSet) for _, is := range ipsetInfo { - proxier.ipsetList[is.name] = NewIPSet(ipset, is.name, is.setType, isIPv6, is.comment) + proxier.ipsetList[is.name] = NewIPSet(ipset, is.name, is.setType, (ipFamily == v1.IPv6Protocol), is.comment) } burstSyncs := 2 klog.V(2).Infof("ipvs(%s) sync params: minSyncPeriod=%v, syncPeriod=%v, burstSyncs=%d", @@ -526,7 +532,7 @@ func NewDualStackProxier( safeIpset := newSafeIpset(ipset) - nodePortAddresses4, nodePortAddresses6 := utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, false) + nodePortAddresses4, nodePortAddresses6 := utilproxy.FilterIncorrectCIDRVersion(nodePortAddresses, v1.IPv4Protocol) // Create an ipv4 instance of the single-stack proxier ipv4Proxier, err := NewProxier(ipt[0], ipvs, safeIpset, sysctl, @@ -1149,6 +1155,17 @@ func (proxier *Proxier) syncProxyRules() { } } + // filter node IPs by proxier ipfamily + idx := 0 + for _, nodeIP := range nodeIPs { + if (proxier.ipFamily == v1.IPv6Protocol) == utilnet.IsIPv6(nodeIP) { + nodeIPs[idx] = nodeIP + idx++ + } + } + // reset slice to filtered entries + nodeIPs = nodeIPs[:idx] + // Build IPVS rules for each service. for svcName, svc := range proxier.serviceMap { svcInfo, ok := svc.(*serviceInfo) diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 93d7ecdff1a..39e87f407d0 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -49,6 +49,8 @@ import ( "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" utilpointer "k8s.io/utils/pointer" + + utilnet "k8s.io/utils/net" ) const testHostname = "test-hostname" @@ -102,7 +104,22 @@ func (fake *fakeIPSetVersioner) GetVersion() (string, error) { return fake.version, fake.err } -func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP, excludeCIDRs []*net.IPNet, endpointSlicesEnabled bool) *Proxier { +func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP, excludeCIDRs []*net.IPNet, endpointSlicesEnabled bool, ipFamily v1.IPFamily) *Proxier { + // unlike actual proxier, this fake proxier does not filter node IPs per family requested + // which can lead to false postives. + + // filter node IPs by proxier ipfamily + idx := 0 + for _, nodeIP := range nodeIPs { + if (ipFamily == v1.IPv6Protocol) == utilnet.IsIPv6(nodeIP) { + nodeIPs[idx] = nodeIP + idx++ + } + } + + // reset slice to filtered entries + nodeIPs = nodeIPs[:idx] + fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte("dummy device have been created"), nil, nil }, @@ -124,9 +141,9 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u p := &Proxier{ exec: fexec, serviceMap: make(proxy.ServiceMap), - serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, nil, nil, nil), + serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, ipFamily, nil, nil), endpointsMap: make(proxy.EndpointsMap), - endpointsChanges: proxy.NewEndpointChangeTracker(testHostname, nil, nil, nil, endpointSlicesEnabled, nil), + endpointsChanges: proxy.NewEndpointChangeTracker(testHostname, nil, ipFamily, nil, endpointSlicesEnabled, nil), excludeCIDRs: excludeCIDRs, iptables: ipt, ipvs: ipvs, @@ -150,6 +167,7 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u nodePortAddresses: make([]string, 0), networkInterfacer: proxyutiltest.NewFakeNetwork(), gracefuldeleteManager: NewGracefulTerminationManager(ipvs), + ipFamily: ipFamily, } p.setInitialized(true) p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1) @@ -209,7 +227,7 @@ func TestCleanupLeftovers(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 @@ -442,7 +460,7 @@ func TestGetNodeIPs(t *testing.T) { } } -func TestNodePort(t *testing.T) { +func TestNodePortIPv4(t *testing.T) { tests := []struct { name string services []*v1.Service @@ -510,16 +528,6 @@ func TestNodePort(t *testing.T) { Port: uint16(3001), Scheduler: "rr", }, - { - IP: "2001:db8::1:1", - Port: 3001, - Protocol: "TCP", - }: { - Address: net.ParseIP("2001:db8::1:1"), - Protocol: "TCP", - Port: uint16(3001), - Scheduler: "rr", - }, }, Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ { @@ -532,11 +540,6 @@ func TestNodePort(t *testing.T) { Port: uint16(80), Weight: 1, }, - { - Address: net.ParseIP("1002:ab8::2:10"), - Port: uint16(80), - Weight: 1, - }, }, { IP: "100.101.102.103", @@ -548,27 +551,6 @@ func TestNodePort(t *testing.T) { Port: uint16(80), Weight: 1, }, - { - Address: net.ParseIP("1002:ab8::2:10"), - Port: uint16(80), - Weight: 1, - }, - }, - { - IP: "2001:db8::1:1", - Port: 3001, - Protocol: "TCP", - }: { - { - Address: net.ParseIP("10.180.0.1"), - Port: uint16(80), - Weight: 1, - }, - { - Address: net.ParseIP("1002:ab8::2:10"), - Port: uint16(80), - Weight: 1, - }, }, }, }, @@ -804,36 +786,6 @@ func TestNodePort(t *testing.T) { Port: uint16(3001), Scheduler: "rr", }, - { - IP: "2001:db8::1:1", - Port: 3001, - Protocol: "SCTP", - }: { - Address: net.ParseIP("2001:db8::1:1"), - Protocol: "SCTP", - Port: uint16(3001), - Scheduler: "rr", - }, - { - IP: "2001:db8::1:2", - Port: 3001, - Protocol: "SCTP", - }: { - Address: net.ParseIP("2001:db8::1:2"), - Protocol: "SCTP", - Port: uint16(3001), - Scheduler: "rr", - }, - { - IP: "2001:db8::1:3", - Port: 3001, - Protocol: "SCTP", - }: { - Address: net.ParseIP("2001:db8::1:3"), - Protocol: "SCTP", - Port: uint16(3001), - Scheduler: "rr", - }, }, Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ { @@ -880,39 +832,6 @@ func TestNodePort(t *testing.T) { Weight: 1, }, }, - { - IP: "2001:db8::1:1", - Port: 3001, - Protocol: "SCTP", - }: { - { - Address: net.ParseIP("10.180.0.1"), - Port: uint16(80), - Weight: 1, - }, - }, - { - IP: "2001:db8::1:2", - Port: 3001, - Protocol: "SCTP", - }: { - { - Address: net.ParseIP("10.180.0.1"), - Port: uint16(80), - Weight: 1, - }, - }, - { - IP: "2001:db8::1:3", - Port: 3001, - Protocol: "SCTP", - }: { - { - Address: net.ParseIP("10.180.0.1"), - Port: uint16(80), - Weight: 1, - }, - }, }, }, expectedIPSets: netlinktest.ExpectedIPSet{ @@ -935,24 +854,6 @@ func TestNodePort(t *testing.T) { Protocol: strings.ToLower(string(v1.ProtocolSCTP)), SetType: utilipset.HashIPPort, }, - { - IP: "2001:db8::1:1", - Port: 3001, - Protocol: strings.ToLower(string(v1.ProtocolSCTP)), - SetType: utilipset.HashIPPort, - }, - { - IP: "2001:db8::1:2", - Port: 3001, - Protocol: strings.ToLower(string(v1.ProtocolSCTP)), - SetType: utilipset.HashIPPort, - }, - { - IP: "2001:db8::1:3", - Port: 3001, - Protocol: strings.ToLower(string(v1.ProtocolSCTP)), - SetType: utilipset.HashIPPort, - }, }, }, }, @@ -963,7 +864,7 @@ func TestNodePort(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil, false, v1.IPv4Protocol) fp.nodePortAddresses = test.nodePortAddresses makeServiceMap(fp, test.services...) @@ -972,8 +873,8 @@ func TestNodePort(t *testing.T) { fp.syncProxyRules() if !reflect.DeepEqual(ipvs, test.expectedIPVS) { - t.Logf("actual ipvs state: %v", ipvs) - t.Logf("expected ipvs state: %v", test.expectedIPVS) + t.Logf("actual ipvs state: %+v", ipvs) + t.Logf("expected ipvs state: %+v", test.expectedIPVS) t.Errorf("unexpected IPVS state") } @@ -988,7 +889,354 @@ func TestNodePort(t *testing.T) { } } -func TestClusterIP(t *testing.T) { +func TestNodePortIPv6(t *testing.T) { + tests := []struct { + name string + services []*v1.Service + endpoints []*v1.Endpoints + nodeIPs []net.IP + nodePortAddresses []string + expectedIPVS *ipvstest.FakeIPVS + expectedIPSets netlinktest.ExpectedIPSet + expectedIptablesChains netlinktest.ExpectedIptablesChain + }{ + { + name: "1 service with node port, has 2 endpoints", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = "2020::1" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + NodePort: int32(3001), + }} + }), + }, + endpoints: []*v1.Endpoints{ + makeTestEndpoints("ns1", "svc1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: "10.180.0.1", + }, { + IP: "1002:ab8::2:10", + }}, + Ports: []v1.EndpointPort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + }}, + }} + }), + }, + nodeIPs: []net.IP{ + net.ParseIP("100.101.102.103"), + net.ParseIP("2001:db8::1:1"), + }, + nodePortAddresses: []string{}, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "TCP", + }: { + Address: net.ParseIP("2001:db8::1:1"), + Protocol: "TCP", + Port: uint16(3001), + Scheduler: "rr", + }, + { + IP: "2020::1", + Port: 80, + Protocol: "TCP", + }: { + Address: net.ParseIP("2020::1"), + Protocol: "TCP", + Port: uint16(80), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "TCP", + }: { + { + Address: net.ParseIP("1002:ab8::2:10"), + Port: uint16(80), + Weight: 1, + }, + }, + + { + IP: "2020::1", + Port: 80, + Protocol: "TCP", + }: { + { + Address: net.ParseIP("1002:ab8::2:10"), + Port: uint16(80), + Weight: 1, + }, + }, + }, + }, + }, + + { + name: "1 UDP service with node port, has endpoints (no action on IPv6 Proxier)", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = "10.20.30.41" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolUDP, + NodePort: int32(3001), + }} + }), + }, + endpoints: []*v1.Endpoints{ + makeTestEndpoints("ns1", "svc1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: "10.180.0.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolUDP, + }}, + }} + }), + }, + nodeIPs: []net.IP{ + net.ParseIP("100.101.102.103"), + }, + nodePortAddresses: []string{"0.0.0.0/0"}, + /*since this is a node with only IPv4, proxier should not do anything */ + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{}, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{}, + }, + expectedIPSets: nil, + expectedIptablesChains: nil, + }, + + { + name: "service has node port but no endpoints", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = "2020::1" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + NodePort: int32(3001), + }} + }), + }, + endpoints: []*v1.Endpoints{}, + nodeIPs: []net.IP{ + net.ParseIP("100.101.102.103"), + net.ParseIP("2001:db8::1:1"), + }, + nodePortAddresses: []string{}, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "TCP", + }: { + Address: net.ParseIP("2001:db8::1:1"), + Protocol: "TCP", + Port: uint16(3001), + Scheduler: "rr", + }, + { + IP: "2020::1", + Port: 80, + Protocol: "TCP", + }: { + Address: net.ParseIP("2020::1"), + Protocol: "TCP", + Port: uint16(80), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "2020::1", + Port: 80, + Protocol: "TCP", + }: {}, // no real servers corresponding to no endpoints + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "TCP", + }: {}, // no real servers corresponding to no endpoints + }, + }, + }, + + { + name: "node port service with protocol sctp on a node with multiple nodeIPs", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = "2020::1" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolSCTP, + NodePort: int32(3001), + }} + }), + }, + endpoints: []*v1.Endpoints{ + makeTestEndpoints("ns1", "svc1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: "2001::1", + }}, + Ports: []v1.EndpointPort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolSCTP, + }}, + }} + }), + }, + nodeIPs: []net.IP{ + net.ParseIP("2001:db8::1:1"), + net.ParseIP("2001:db8::1:2"), + }, + nodePortAddresses: []string{}, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "SCTP", + }: { + Address: net.ParseIP("2001:db8::1:1"), + Protocol: "SCTP", + Port: uint16(3001), + Scheduler: "rr", + }, + { + IP: "2001:db8::1:2", + Port: 3001, + Protocol: "SCTP", + }: { + Address: net.ParseIP("2001:db8::1:2"), + Protocol: "SCTP", + Port: uint16(3001), + Scheduler: "rr", + }, + { + IP: "2020::1", + Port: 80, + Protocol: "SCTP", + }: { + Address: net.ParseIP("2020::1"), + Protocol: "SCTP", + Port: uint16(80), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "SCTP", + }: { + { + Address: net.ParseIP("2001::1"), + Port: uint16(80), + Weight: 1, + }, + }, + { + IP: "2001:db8::1:2", + Port: 3001, + Protocol: "SCTP", + }: { + { + Address: net.ParseIP("2001::1"), + Port: uint16(80), + Weight: 1, + }, + }, + { + IP: "2020::1", + Port: 80, + Protocol: "SCTP", + }: { + { + Address: net.ParseIP("2001::1"), + Port: uint16(80), + Weight: 1, + }, + }, + }, + }, + expectedIPSets: netlinktest.ExpectedIPSet{ + kubeNodePortSetSCTP: { + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: strings.ToLower(string(v1.ProtocolSCTP)), + SetType: utilipset.HashIPPort, + }, + { + IP: "2001:db8::1:2", + Port: 3001, + Protocol: strings.ToLower(string(v1.ProtocolSCTP)), + SetType: utilipset.HashIPPort, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ipt := iptablestest.NewFake() + ipvs := ipvstest.NewFake() + ipset := ipsettest.NewFake(testIPSetVersion) + fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil, false, v1.IPv6Protocol) + fp.nodePortAddresses = test.nodePortAddresses + + makeServiceMap(fp, test.services...) + makeEndpointsMap(fp, test.endpoints...) + + fp.syncProxyRules() + + if !reflect.DeepEqual(ipvs, test.expectedIPVS) { + t.Logf("actual ipvs state: %+v", ipvs) + t.Logf("expected ipvs state: %+v", test.expectedIPVS) + t.Errorf("unexpected IPVS state") + } + + if test.expectedIPSets != nil { + checkIPSet(t, fp, test.expectedIPSets) + } + + if test.expectedIptablesChains != nil { + checkIptables(t, ipt, test.expectedIptablesChains) + } + }) + } +} + +func TestIPv4Proxier(t *testing.T) { tests := []struct { name string services []*v1.Service @@ -1053,16 +1301,6 @@ func TestClusterIP(t *testing.T) { Port: uint16(80), Scheduler: "rr", }, - { - IP: "1002:ab8::2:1", - Port: 8080, - Protocol: "TCP", - }: { - Address: net.ParseIP("1002:ab8::2:1"), - Protocol: "TCP", - Port: uint16(8080), - Scheduler: "rr", - }, }, Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ { @@ -1076,17 +1314,6 @@ func TestClusterIP(t *testing.T) { Weight: 1, }, }, - { - IP: "1002:ab8::2:1", - Port: 8080, - Protocol: "TCP", - }: { - { - Address: net.ParseIP("1009:ab8::5:6"), - Port: uint16(8080), - Weight: 1, - }, - }, }, }, }, @@ -1132,7 +1359,146 @@ func TestClusterIP(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) + + makeServiceMap(fp, test.services...) + makeEndpointsMap(fp, test.endpoints...) + + fp.syncProxyRules() + + if !reflect.DeepEqual(ipvs, test.expectedIPVS) { + t.Logf("actual ipvs state: %v", ipvs) + t.Logf("expected ipvs state: %v", test.expectedIPVS) + t.Errorf("unexpected IPVS state") + } + }) + } +} + +func TestIPv6Proxier(t *testing.T) { + tests := []struct { + name string + services []*v1.Service + endpoints []*v1.Endpoints + expectedIPVS *ipvstest.FakeIPVS + }{ + { + name: "2 services with Cluster IP, each with endpoints", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.ClusterIP = "10.20.30.41" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + }} + }), + makeTestService("ns2", "svc2", func(svc *v1.Service) { + svc.Spec.ClusterIP = "1002:ab8::2:1" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p8080", + Port: int32(8080), + Protocol: v1.ProtocolTCP, + }} + }), + }, + endpoints: []*v1.Endpoints{ + makeTestEndpoints("ns1", "svc1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: "10.180.0.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + }}, + }} + }), + makeTestEndpoints("ns2", "svc2", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: "1009:ab8::5:6", + }}, + Ports: []v1.EndpointPort{{ + Name: "p8080", + Port: int32(8080), + Protocol: v1.ProtocolTCP, + }}, + }} + }), + }, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1002:ab8::2:1", + Port: 8080, + Protocol: "TCP", + }: { + Address: net.ParseIP("1002:ab8::2:1"), + Protocol: "TCP", + Port: uint16(8080), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1002:ab8::2:1", + Port: 8080, + Protocol: "TCP", + }: { + { + Address: net.ParseIP("1009:ab8::5:6"), + Port: uint16(8080), + Weight: 1, + }, + }, + }, + }, + }, + { + name: "cluster IP service with no endpoints", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.ClusterIP = "2001::1" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + }} + }), + }, + endpoints: []*v1.Endpoints{}, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "2001::1", + Port: 80, + Protocol: "TCP", + }: { + Address: net.ParseIP("2001::1"), + Protocol: "TCP", + Port: uint16(80), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "2001::1", + Port: 80, + Protocol: "TCP", + }: {}, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ipt := iptablestest.NewFake() + ipvs := ipvstest.NewFake() + ipset := ipsettest.NewFake(testIPSetVersion) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv6Protocol) makeServiceMap(fp, test.services...) makeEndpointsMap(fp, test.endpoints...) @@ -1153,7 +1519,7 @@ func TestMasqueradeRule(t *testing.T) { ipt := iptablestest.NewFake().SetHasRandomFully(testcase) ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) makeServiceMap(fp) makeEndpointsMap(fp) fp.syncProxyRules() @@ -1173,7 +1539,7 @@ func TestExternalIPsNoEndpoint(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) svcIP := "10.20.30.41" svcPort := 80 svcExternalIPs := "50.60.70.81" @@ -1228,7 +1594,7 @@ func TestExternalIPs(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) svcIP := "10.20.30.41" svcPort := 80 svcExternalIPs := sets.NewString("50.60.70.81", "2012::51", "127.0.0.1") @@ -1273,8 +1639,8 @@ func TestExternalIPs(t *testing.T) { if err != nil { t.Errorf("Failed to get ipvs services, err: %v", err) } - if len(services) != 4 { - t.Errorf("Expect 4 ipvs services, got %d", len(services)) + if len(services) != 3 { // ipvs filters out by ipfamily + t.Errorf("Expect 3 ipvs services, got %d", len(services)) } found := false for _, svc := range services { @@ -1298,7 +1664,7 @@ func TestOnlyLocalExternalIPs(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) svcIP := "10.20.30.41" svcPort := 80 svcExternalIPs := sets.NewString("50.60.70.81", "2012::51", "127.0.0.1") @@ -1353,8 +1719,8 @@ func TestOnlyLocalExternalIPs(t *testing.T) { if err != nil { t.Errorf("Failed to get ipvs services, err: %v", err) } - if len(services) != 4 { - t.Errorf("Expect 4 ipvs services, got %d", len(services)) + if len(services) != 3 { // ipvs filters out by IPFamily + t.Errorf("Expect 3 ipvs services, got %d", len(services)) } found := false for _, svc := range services { @@ -1520,9 +1886,9 @@ func TestOnlyLocalNodePorts(t *testing.T) { fp.syncProxyRules() - // Expect 3 services and 1 destination + // Expect 2 (matching ipvs IPFamily field) services and 1 destination epVS := &netlinktest.ExpectedVirtualServer{ - VSNum: 3, IP: nodeIP.String(), Port: uint16(svcNodePort), Protocol: string(v1.ProtocolTCP), + VSNum: 2, IP: nodeIP.String(), Port: uint16(svcNodePort), Protocol: string(v1.ProtocolTCP), RS: []netlinktest.ExpectedRealServer{{ IP: epIP, Port: uint16(svcPort), }}} @@ -1835,7 +2201,7 @@ func TestBuildServiceMapAddRemove(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) services := []*v1.Service{ makeTestService("somewhere-else", "cluster-ip", func(svc *v1.Service) { @@ -1945,7 +2311,7 @@ func TestBuildServiceMapServiceHeadless(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) makeServiceMap(fp, makeTestService("somewhere-else", "headless", func(svc *v1.Service) { @@ -1984,7 +2350,7 @@ func TestBuildServiceMapServiceTypeExternalName(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) makeServiceMap(fp, makeTestService("somewhere-else", "external-name", func(svc *v1.Service) { @@ -2012,7 +2378,7 @@ func TestBuildServiceMapServiceUpdate(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) servicev1 := makeTestService("somewhere", "some-service", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP @@ -2096,7 +2462,7 @@ func TestSessionAffinity(t *testing.T) { ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) nodeIP := net.ParseIP("100.101.102.103") - fp := NewFakeProxier(ipt, ipvs, ipset, []net.IP{nodeIP}, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, []net.IP{nodeIP}, nil, false, v1.IPv4Protocol) svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 @@ -2991,7 +3357,7 @@ func Test_updateEndpointsMap(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) fp.hostname = nodeName // First check that after adding all previous versions of endpoints, @@ -3264,7 +3630,7 @@ func Test_syncService(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - proxier := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + proxier := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) proxier.netlinkHandle.EnsureDummyDevice(DefaultDummyDevice) if testCases[i].oldVirtualServer != nil { @@ -3294,7 +3660,7 @@ func buildFakeProxier() (*iptablestest.FakeIPTables, *Proxier) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - return ipt, NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + return ipt, NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) } func hasJump(rules []iptablestest.Rule, destChain, ipSet string) bool { @@ -3354,6 +3720,7 @@ func checkIPSet(t *testing.T, fp *Proxier, ipSet netlinktest.ExpectedIPSet) { // checkIPVS to check expected ipvs service and destination func checkIPVS(t *testing.T, fp *Proxier, vs *netlinktest.ExpectedVirtualServer) { + t.Helper() services, err := fp.ipvs.GetVirtualServers() if err != nil { t.Errorf("Failed to get ipvs services, err: %v", err) @@ -3380,7 +3747,7 @@ func TestCleanLegacyService(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, parseExcludedCIDRs([]string{"3.3.3.0/24", "4.4.4.0/24"}), false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, parseExcludedCIDRs([]string{"3.3.3.0/24", "4.4.4.0/24"}), false, v1.IPv4Protocol) // All ipvs services that were processed in the latest sync loop. activeServices := map[string]bool{"ipvs0": true, "ipvs1": true} @@ -3486,7 +3853,7 @@ func TestCleanLegacyServiceWithRealServers(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) // all deleted expect ipvs2 activeServices := map[string]bool{"ipvs2": true} @@ -3580,7 +3947,7 @@ func TestCleanLegacyRealServersExcludeCIDRs(t *testing.T) { ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) gtm := NewGracefulTerminationManager(ipvs) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, parseExcludedCIDRs([]string{"4.4.4.4/32"}), false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, parseExcludedCIDRs([]string{"4.4.4.4/32"}), false, v1.IPv4Protocol) fp.gracefuldeleteManager = gtm vs := &utilipvs.VirtualServer{ @@ -3634,7 +4001,7 @@ func TestCleanLegacyService6(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, parseExcludedCIDRs([]string{"3000::/64", "4000::/64"}), false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, parseExcludedCIDRs([]string{"3000::/64", "4000::/64"}), false, v1.IPv4Protocol) fp.nodeIP = net.ParseIP("::1") // All ipvs services that were processed in the latest sync loop. @@ -3741,7 +4108,7 @@ func TestMultiPortServiceBindAddr(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false, v1.IPv4Protocol) service1 := makeTestService("ns1", "svc1", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP @@ -3846,7 +4213,7 @@ func TestEndpointSliceE2E(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, true) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, true, v1.IPv4Protocol) fp.servicesSynced = true fp.endpointsSynced = true fp.endpointSlicesSynced = true diff --git a/pkg/proxy/metaproxier/BUILD b/pkg/proxy/metaproxier/BUILD index 8960baf81a7..07ec2c8a99a 100644 --- a/pkg/proxy/metaproxier/BUILD +++ b/pkg/proxy/metaproxier/BUILD @@ -9,7 +9,6 @@ go_library( deps = [ "//pkg/proxy:go_default_library", "//pkg/proxy/config:go_default_library", - "//pkg/proxy/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/discovery/v1beta1:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", diff --git a/pkg/proxy/metaproxier/meta_proxier.go b/pkg/proxy/metaproxier/meta_proxier.go index 9dfdfdb02ee..20b0413ea0f 100644 --- a/pkg/proxy/metaproxier/meta_proxier.go +++ b/pkg/proxy/metaproxier/meta_proxier.go @@ -20,18 +20,18 @@ import ( "fmt" v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1beta1" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/config" - utilproxy "k8s.io/kubernetes/pkg/proxy/util" utilnet "k8s.io/utils/net" - - discovery "k8s.io/api/discovery/v1beta1" ) type metaProxier struct { + // actual, wrapped ipv4Proxier proxy.Provider + // actual, wrapped ipv6Proxier proxy.Provider // TODO(imroc): implement node handler for meta proxier. config.NoopNodeHandler @@ -63,41 +63,23 @@ func (proxier *metaProxier) SyncLoop() { // OnServiceAdd is called whenever creation of new service object is observed. func (proxier *metaProxier) OnServiceAdd(service *v1.Service) { - if utilproxy.ShouldSkipService(service) { - return - } - if utilnet.IsIPv6String(service.Spec.ClusterIP) { - proxier.ipv6Proxier.OnServiceAdd(service) - } else { - proxier.ipv4Proxier.OnServiceAdd(service) - } + proxier.ipv4Proxier.OnServiceAdd(service) + proxier.ipv6Proxier.OnServiceAdd(service) } // OnServiceUpdate is called whenever modification of an existing // service object is observed. func (proxier *metaProxier) OnServiceUpdate(oldService, service *v1.Service) { - if utilproxy.ShouldSkipService(service) { - return - } - // IPFamily is immutable, hence we only need to check on the new service - if utilnet.IsIPv6String(service.Spec.ClusterIP) { - proxier.ipv6Proxier.OnServiceUpdate(oldService, service) - } else { - proxier.ipv4Proxier.OnServiceUpdate(oldService, service) - } + proxier.ipv4Proxier.OnServiceUpdate(oldService, service) + proxier.ipv6Proxier.OnServiceUpdate(oldService, service) } // OnServiceDelete is called whenever deletion of an existing service // object is observed. func (proxier *metaProxier) OnServiceDelete(service *v1.Service) { - if utilproxy.ShouldSkipService(service) { - return - } - if utilnet.IsIPv6String(service.Spec.ClusterIP) { - proxier.ipv6Proxier.OnServiceDelete(service) - } else { - proxier.ipv4Proxier.OnServiceDelete(service) - } + proxier.ipv4Proxier.OnServiceDelete(service) + proxier.ipv6Proxier.OnServiceDelete(service) + } // OnServiceSynced is called once all the initial event handlers were @@ -161,8 +143,6 @@ func (proxier *metaProxier) OnEndpointsSynced() { proxier.ipv6Proxier.OnEndpointsSynced() } -// TODO: (khenidak) implement EndpointSlice handling - // OnEndpointSliceAdd is called whenever creation of a new endpoint slice object // is observed. func (proxier *metaProxier) OnEndpointSliceAdd(endpointSlice *discovery.EndpointSlice) { diff --git a/pkg/proxy/service.go b/pkg/proxy/service.go index 48bd402f02c..a00b2ddab1c 100644 --- a/pkg/proxy/service.go +++ b/pkg/proxy/service.go @@ -32,7 +32,6 @@ import ( apiservice "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/proxy/metrics" utilproxy "k8s.io/kubernetes/pkg/proxy/util" - utilnet "k8s.io/utils/net" ) // BaseServiceInfo contains base information that defines a service. @@ -135,8 +134,10 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic // Kube-apiserver side guarantees SessionAffinityConfig won't be nil when session affinity type is ClientIP stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) } + + clusterIP := utilproxy.GetClusterIPByFamily(sct.ipFamily, service) info := &BaseServiceInfo{ - clusterIP: net.ParseIP(service.Spec.ClusterIP), + clusterIP: net.ParseIP(clusterIP), port: int(port.Port), protocol: port.Protocol, nodePort: int(port.NodePort), @@ -150,41 +151,38 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic for i, sourceRange := range service.Spec.LoadBalancerSourceRanges { loadBalancerSourceRanges[i] = strings.TrimSpace(sourceRange) } + // filter external ips, source ranges and ingress ips + // prior to dual stack services, this was considered an error, but with dual stack + // services, this is actually expected. Hence we downgraded from reporting by events + // to just log lines with high verbosity - if sct.isIPv6Mode == nil { - info.externalIPs = make([]string, len(service.Spec.ExternalIPs)) - info.loadBalancerSourceRanges = loadBalancerSourceRanges - copy(info.externalIPs, service.Spec.ExternalIPs) - // Deep-copy in case the service instance changes - info.loadBalancerStatus = *service.Status.LoadBalancer.DeepCopy() - } else { - // Filter out the incorrect IP version case. - // If ExternalIPs, LoadBalancerSourceRanges and LoadBalancerStatus Ingress on service contains incorrect IP versions, - // only filter out the incorrect ones. - var incorrectIPs []string - info.externalIPs, incorrectIPs = utilproxy.FilterIncorrectIPVersion(service.Spec.ExternalIPs, *sct.isIPv6Mode) - if len(incorrectIPs) > 0 { - utilproxy.LogAndEmitIncorrectIPVersionEvent(sct.recorder, "externalIPs", strings.Join(incorrectIPs, ","), service.Namespace, service.Name, service.UID) - } - info.loadBalancerSourceRanges, incorrectIPs = utilproxy.FilterIncorrectCIDRVersion(loadBalancerSourceRanges, *sct.isIPv6Mode) - if len(incorrectIPs) > 0 { - utilproxy.LogAndEmitIncorrectIPVersionEvent(sct.recorder, "loadBalancerSourceRanges", strings.Join(incorrectIPs, ","), service.Namespace, service.Name, service.UID) - } - // Obtain Load Balancer Ingress IPs - var ips []string - for _, ing := range service.Status.LoadBalancer.Ingress { - ips = append(ips, ing.IP) - } - if len(ips) > 0 { - correctIPs, incorrectIPs := utilproxy.FilterIncorrectIPVersion(ips, *sct.isIPv6Mode) - if len(incorrectIPs) > 0 { - utilproxy.LogAndEmitIncorrectIPVersionEvent(sct.recorder, "Load Balancer ingress IPs", strings.Join(incorrectIPs, ","), service.Namespace, service.Name, service.UID) - } - // Create the LoadBalancerStatus with the filtererd IPs - for _, ip := range correctIPs { - info.loadBalancerStatus.Ingress = append(info.loadBalancerStatus.Ingress, v1.LoadBalancerIngress{IP: ip}) + var incorrectIPs []string + info.externalIPs, incorrectIPs = utilproxy.FilterIncorrectIPVersion(service.Spec.ExternalIPs, sct.ipFamily) + if len(incorrectIPs) > 0 { + klog.V(4).Infof("service change tracker(%v) ignored the following external IPs(%s) for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name) + } - } + info.loadBalancerSourceRanges, incorrectIPs = utilproxy.FilterIncorrectCIDRVersion(loadBalancerSourceRanges, sct.ipFamily) + if len(incorrectIPs) > 0 { + klog.V(4).Infof("service change tracker(%v) ignored the following load balancer source ranges(%s) for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name) + } + + // Obtain Load Balancer Ingress IPs + var ips []string + for _, ing := range service.Status.LoadBalancer.Ingress { + ips = append(ips, ing.IP) + } + + if len(ips) > 0 { + correctIPs, incorrectIPs := utilproxy.FilterIncorrectIPVersion(ips, sct.ipFamily) + + if len(incorrectIPs) > 0 { + klog.V(4).Infof("service change tracker(%v) ignored the following load balancer(%s) ingress ips for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name) + + } + // Create the LoadBalancerStatus with the filtered IPs + for _, ip := range correctIPs { + info.loadBalancerStatus.Ingress = append(info.loadBalancerStatus.Ingress, v1.LoadBalancerIngress{IP: ip}) } } @@ -224,18 +222,18 @@ type ServiceChangeTracker struct { // makeServiceInfo allows proxier to inject customized information when processing service. makeServiceInfo makeServicePortFunc processServiceMapChange processServiceMapChangeFunc - // isIPv6Mode indicates if change tracker is under IPv6/IPv4 mode. Nil means not applicable. - isIPv6Mode *bool - recorder record.EventRecorder + ipFamily v1.IPFamily + + recorder record.EventRecorder } // NewServiceChangeTracker initializes a ServiceChangeTracker -func NewServiceChangeTracker(makeServiceInfo makeServicePortFunc, isIPv6Mode *bool, recorder record.EventRecorder, processServiceMapChange processServiceMapChangeFunc) *ServiceChangeTracker { +func NewServiceChangeTracker(makeServiceInfo makeServicePortFunc, ipFamily v1.IPFamily, recorder record.EventRecorder, processServiceMapChange processServiceMapChangeFunc) *ServiceChangeTracker { return &ServiceChangeTracker{ items: make(map[types.NamespacedName]*serviceChange), makeServiceInfo: makeServiceInfo, - isIPv6Mode: isIPv6Mode, recorder: recorder, + ipFamily: ipFamily, processServiceMapChange: processServiceMapChange, } } @@ -322,13 +320,9 @@ func (sct *ServiceChangeTracker) serviceToServiceMap(service *v1.Service) Servic return nil } - if len(service.Spec.ClusterIP) != 0 { - // Filter out the incorrect IP version case. - // If ClusterIP on service has incorrect IP version, service itself will be ignored. - if sct.isIPv6Mode != nil && utilnet.IsIPv6String(service.Spec.ClusterIP) != *sct.isIPv6Mode { - utilproxy.LogAndEmitIncorrectIPVersionEvent(sct.recorder, "clusterIP", service.Spec.ClusterIP, service.Namespace, service.Name, service.UID) - return nil - } + clusterIP := utilproxy.GetClusterIPByFamily(sct.ipFamily, service) + if clusterIP == "" { + return nil } serviceMap := make(ServiceMap) diff --git a/pkg/proxy/service_test.go b/pkg/proxy/service_test.go index d76afdf2a59..a690608a885 100644 --- a/pkg/proxy/service_test.go +++ b/pkg/proxy/service_test.go @@ -85,10 +85,6 @@ func makeServicePortName(ns, name, port string, protocol v1.Protocol) ServicePor } func TestServiceToServiceMap(t *testing.T) { - svcTracker := NewServiceChangeTracker(nil, nil, nil, nil) - - trueVal := true - falseVal := false testClusterIPv4 := "10.0.0.1" testExternalIPv4 := "8.8.8.8" testSourceRangeIPv4 := "0.0.0.0/1" @@ -97,18 +93,22 @@ func TestServiceToServiceMap(t *testing.T) { testSourceRangeIPv6 := "2001:db8::/32" testCases := []struct { - desc string - service *v1.Service - expected map[ServicePortName]*BaseServiceInfo - isIPv6Mode *bool + desc string + service *v1.Service + expected map[ServicePortName]*BaseServiceInfo + ipFamily v1.IPFamily }{ { desc: "nothing", + ipFamily: v1.IPv4Protocol, + service: nil, expected: map[ServicePortName]*BaseServiceInfo{}, }, { - desc: "headless service", + desc: "headless service", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns2", "headless", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP svc.Spec.ClusterIP = v1.ClusterIPNone @@ -117,7 +117,9 @@ func TestServiceToServiceMap(t *testing.T) { expected: map[ServicePortName]*BaseServiceInfo{}, }, { - desc: "headless sctp service", + desc: "headless sctp service", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns2", "headless", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP svc.Spec.ClusterIP = v1.ClusterIPNone @@ -126,7 +128,9 @@ func TestServiceToServiceMap(t *testing.T) { expected: map[ServicePortName]*BaseServiceInfo{}, }, { - desc: "headless service without port", + desc: "headless service without port", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns2", "headless-without-port", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP svc.Spec.ClusterIP = v1.ClusterIPNone @@ -134,7 +138,9 @@ func TestServiceToServiceMap(t *testing.T) { expected: map[ServicePortName]*BaseServiceInfo{}, }, { - desc: "cluster ip service", + desc: "cluster ip service", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns2", "cluster-ip", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP svc.Spec.ClusterIP = "172.16.55.4" @@ -147,7 +153,9 @@ func TestServiceToServiceMap(t *testing.T) { }, }, { - desc: "nodeport service", + desc: "nodeport service", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns2", "node-port", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeNodePort svc.Spec.ClusterIP = "172.16.55.10" @@ -160,7 +168,9 @@ func TestServiceToServiceMap(t *testing.T) { }, }, { - desc: "load balancer service", + desc: "load balancer service", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns1", "load-balancer", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ClusterIP = "172.16.55.11" @@ -179,7 +189,9 @@ func TestServiceToServiceMap(t *testing.T) { }, }, { - desc: "load balancer service with only local traffic policy", + desc: "load balancer service with only local traffic policy", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns1", "only-local-load-balancer", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ClusterIP = "172.16.55.12" @@ -200,7 +212,9 @@ func TestServiceToServiceMap(t *testing.T) { }, }, { - desc: "external name service", + desc: "external name service", + ipFamily: v1.IPv4Protocol, + service: makeTestService("ns2", "external-name", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeExternalName svc.Spec.ClusterIP = "172.16.55.4" // Should be ignored @@ -210,7 +224,9 @@ func TestServiceToServiceMap(t *testing.T) { expected: map[ServicePortName]*BaseServiceInfo{}, }, { - desc: "service with ipv6 clusterIP under ipv4 mode, service should be filtered", + desc: "service with ipv6 clusterIP under ipv4 mode, service should be filtered", + ipFamily: v1.IPv4Protocol, + service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "invalidIPv6InIPV4Mode", @@ -235,10 +251,11 @@ func TestServiceToServiceMap(t *testing.T) { }, }, }, - isIPv6Mode: &falseVal, }, { - desc: "service with ipv4 clusterIP under ipv6 mode, service should be filtered", + desc: "service with ipv4 clusterIP under ipv6 mode, service should be filtered", + ipFamily: v1.IPv6Protocol, + service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "invalidIPv4InIPV6Mode", @@ -263,10 +280,11 @@ func TestServiceToServiceMap(t *testing.T) { }, }, }, - isIPv6Mode: &trueVal, }, { - desc: "service with ipv4 configurations under ipv4 mode", + desc: "service with ipv4 configurations under ipv4 mode", + ipFamily: v1.IPv4Protocol, + service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "validIPv4", @@ -300,10 +318,11 @@ func TestServiceToServiceMap(t *testing.T) { info.loadBalancerStatus.Ingress = []v1.LoadBalancerIngress{{IP: testExternalIPv4}} }), }, - isIPv6Mode: &falseVal, }, { - desc: "service with ipv6 configurations under ipv6 mode", + desc: "service with ipv6 configurations under ipv6 mode", + ipFamily: v1.IPv6Protocol, + service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "validIPv6", @@ -337,10 +356,11 @@ func TestServiceToServiceMap(t *testing.T) { info.loadBalancerStatus.Ingress = []v1.LoadBalancerIngress{{IP: testExternalIPv6}} }), }, - isIPv6Mode: &trueVal, }, { - desc: "service with both ipv4 and ipv6 configurations under ipv4 mode, ipv6 fields should be filtered", + desc: "service with both ipv4 and ipv6 configurations under ipv4 mode, ipv6 fields should be filtered", + ipFamily: v1.IPv4Protocol, + service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "filterIPv6InIPV4Mode", @@ -374,10 +394,11 @@ func TestServiceToServiceMap(t *testing.T) { info.loadBalancerStatus.Ingress = []v1.LoadBalancerIngress{{IP: testExternalIPv4}} }), }, - isIPv6Mode: &falseVal, }, { - desc: "service with both ipv4 and ipv6 configurations under ipv6 mode, ipv4 fields should be filtered", + desc: "service with both ipv4 and ipv6 configurations under ipv6 mode, ipv4 fields should be filtered", + ipFamily: v1.IPv6Protocol, + service: &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "filterIPv4InIPV6Mode", @@ -411,7 +432,6 @@ func TestServiceToServiceMap(t *testing.T) { info.loadBalancerStatus.Ingress = []v1.LoadBalancerIngress{{IP: testExternalIPv6}} }), }, - isIPv6Mode: &trueVal, }, { desc: "service with extra space in LoadBalancerSourceRanges", @@ -437,21 +457,24 @@ func TestServiceToServiceMap(t *testing.T) { info.loadBalancerSourceRanges = []string{"10.1.2.0/28"} }), }, - isIPv6Mode: &falseVal, }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - svcTracker.isIPv6Mode = tc.isIPv6Mode + svcTracker := NewServiceChangeTracker(nil, tc.ipFamily, nil, nil) // outputs newServices := svcTracker.serviceToServiceMap(tc.service) if len(newServices) != len(tc.expected) { - t.Errorf("expected %d new, got %d: %v", len(tc.expected), len(newServices), spew.Sdump(newServices)) + t.Fatalf("expected %d new, got %d: %v", len(tc.expected), len(newServices), spew.Sdump(newServices)) } for svcKey, expectedInfo := range tc.expected { - svcInfo, _ := newServices[svcKey].(*BaseServiceInfo) + svcInfo, exists := newServices[svcKey].(*BaseServiceInfo) + if !exists { + t.Fatalf("[%s] expected to find key %s", tc.desc, svcKey) + } + if !svcInfo.clusterIP.Equal(expectedInfo.clusterIP) || svcInfo.port != expectedInfo.port || svcInfo.protocol != expectedInfo.protocol || @@ -459,7 +482,19 @@ func TestServiceToServiceMap(t *testing.T) { !sets.NewString(svcInfo.externalIPs...).Equal(sets.NewString(expectedInfo.externalIPs...)) || !sets.NewString(svcInfo.loadBalancerSourceRanges...).Equal(sets.NewString(expectedInfo.loadBalancerSourceRanges...)) || !reflect.DeepEqual(svcInfo.loadBalancerStatus, expectedInfo.loadBalancerStatus) { - t.Errorf("expected new[%v]to be %v, got %v", svcKey, expectedInfo, *svcInfo) + t.Errorf("[%s] expected new[%v]to be %v, got %v", tc.desc, svcKey, expectedInfo, *svcInfo) + } + for svcKey, expectedInfo := range tc.expected { + svcInfo, _ := newServices[svcKey].(*BaseServiceInfo) + if !svcInfo.clusterIP.Equal(expectedInfo.clusterIP) || + svcInfo.port != expectedInfo.port || + svcInfo.protocol != expectedInfo.protocol || + svcInfo.healthCheckNodePort != expectedInfo.healthCheckNodePort || + !sets.NewString(svcInfo.externalIPs...).Equal(sets.NewString(expectedInfo.externalIPs...)) || + !sets.NewString(svcInfo.loadBalancerSourceRanges...).Equal(sets.NewString(expectedInfo.loadBalancerSourceRanges...)) || + !reflect.DeepEqual(svcInfo.loadBalancerStatus, expectedInfo.loadBalancerStatus) { + t.Errorf("expected new[%v]to be %v, got %v", svcKey, expectedInfo, *svcInfo) + } } } }) @@ -474,12 +509,12 @@ type FakeProxier struct { hostname string } -func newFakeProxier() *FakeProxier { +func newFakeProxier(ipFamily v1.IPFamily) *FakeProxier { return &FakeProxier{ serviceMap: make(ServiceMap), - serviceChanges: NewServiceChangeTracker(nil, nil, nil, nil), + serviceChanges: NewServiceChangeTracker(nil, ipFamily, nil, nil), endpointsMap: make(EndpointsMap), - endpointsChanges: NewEndpointChangeTracker(testHostname, nil, nil, nil, false, nil), + endpointsChanges: NewEndpointChangeTracker(testHostname, nil, ipFamily, nil, false, nil), } } @@ -502,7 +537,7 @@ func (fake *FakeProxier) deleteService(service *v1.Service) { } func TestUpdateServiceMapHeadless(t *testing.T) { - fp := newFakeProxier() + fp := newFakeProxier(v1.IPv4Protocol) makeServiceMap(fp, makeTestService("ns2", "headless", func(svc *v1.Service) { @@ -533,7 +568,7 @@ func TestUpdateServiceMapHeadless(t *testing.T) { } func TestUpdateServiceTypeExternalName(t *testing.T) { - fp := newFakeProxier() + fp := newFakeProxier(v1.IPv4Protocol) makeServiceMap(fp, makeTestService("ns2", "external-name", func(svc *v1.Service) { @@ -558,7 +593,7 @@ func TestUpdateServiceTypeExternalName(t *testing.T) { } func TestBuildServiceMapAddRemove(t *testing.T) { - fp := newFakeProxier() + fp := newFakeProxier(v1.IPv4Protocol) services := []*v1.Service{ makeTestService("ns2", "cluster-ip", func(svc *v1.Service) { @@ -661,7 +696,7 @@ func TestBuildServiceMapAddRemove(t *testing.T) { } func TestBuildServiceMapServiceUpdate(t *testing.T) { - fp := newFakeProxier() + fp := newFakeProxier(v1.IPv4Protocol) servicev1 := makeTestService("ns1", "svc1", func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index 4cf4e21eaba..2031cd1a09e 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -256,13 +256,13 @@ func LogAndEmitIncorrectIPVersionEvent(recorder record.EventRecorder, fieldName, } // FilterIncorrectIPVersion filters out the incorrect IP version case from a slice of IP strings. -func FilterIncorrectIPVersion(ipStrings []string, isIPv6Mode bool) ([]string, []string) { - return filterWithCondition(ipStrings, isIPv6Mode, utilnet.IsIPv6String) +func FilterIncorrectIPVersion(ipStrings []string, ipfamily v1.IPFamily) ([]string, []string) { + return filterWithCondition(ipStrings, (ipfamily == v1.IPv6Protocol), utilnet.IsIPv6String) } // FilterIncorrectCIDRVersion filters out the incorrect IP version case from a slice of CIDR strings. -func FilterIncorrectCIDRVersion(ipStrings []string, isIPv6Mode bool) ([]string, []string) { - return filterWithCondition(ipStrings, isIPv6Mode, utilnet.IsIPv6CIDRString) +func FilterIncorrectCIDRVersion(ipStrings []string, ipfamily v1.IPFamily) ([]string, []string) { + return filterWithCondition(ipStrings, (ipfamily == v1.IPv6Protocol), utilnet.IsIPv6CIDRString) } func filterWithCondition(strs []string, expectedCondition bool, conditionFunc func(string) bool) ([]string, []string) { @@ -376,3 +376,30 @@ func NewFilteredDialContext(wrapped DialContext, resolv Resolver, opts *Filtered return wrapped(ctx, network, address) } } + +// GetClusterIPByFamily returns a service clusterip by family +func GetClusterIPByFamily(ipFamily v1.IPFamily, service *v1.Service) string { + // allowing skew + if len(service.Spec.IPFamilies) == 0 { + if len(service.Spec.ClusterIP) == 0 || service.Spec.ClusterIP == v1.ClusterIPNone { + return "" + } + + IsIPv6Family := (ipFamily == v1.IPv6Protocol) + if IsIPv6Family == utilnet.IsIPv6String(service.Spec.ClusterIP) { + return service.Spec.ClusterIP + } + + return "" + } + + for idx, family := range service.Spec.IPFamilies { + if family == ipFamily { + if idx < len(service.Spec.ClusterIPs) { + return service.Spec.ClusterIPs[idx] + } + } + } + + return "" +} diff --git a/pkg/proxy/util/utils_test.go b/pkg/proxy/util/utils_test.go index e7fa4dcea54..f10a95c88b3 100644 --- a/pkg/proxy/util/utils_test.go +++ b/pkg/proxy/util/utils_test.go @@ -649,7 +649,11 @@ func TestFilterIncorrectIPVersion(t *testing.T) { for _, testcase := range testCases { t.Run(testcase.desc, func(t *testing.T) { - correct, incorrect := FilterIncorrectIPVersion(testcase.ipString, testcase.wantIPv6) + ipFamily := v1.IPv4Protocol + if testcase.wantIPv6 { + ipFamily = v1.IPv6Protocol + } + correct, incorrect := FilterIncorrectIPVersion(testcase.ipString, ipFamily) if !reflect.DeepEqual(testcase.expectCorrect, correct) { t.Errorf("Test %v failed: expected %v, got %v", testcase.desc, testcase.expectCorrect, correct) } @@ -659,3 +663,162 @@ func TestFilterIncorrectIPVersion(t *testing.T) { }) } } + +func TestGetClusterIPByFamily(t *testing.T) { + testCases := []struct { + name string + service v1.Service + requestFamily v1.IPFamily + expectedResult string + }{ + { + name: "old style service ipv4. want ipv4", + requestFamily: v1.IPv4Protocol, + expectedResult: "10.0.0.10", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: "10.0.0.10", + }, + }, + }, + + { + name: "old style service ipv4. want ipv6", + requestFamily: v1.IPv6Protocol, + expectedResult: "", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: "10.0.0.10", + }, + }, + }, + + { + name: "old style service ipv6. want ipv6", + requestFamily: v1.IPv6Protocol, + expectedResult: "2000::1", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: "2000::1", + }, + }, + }, + + { + name: "old style service ipv6. want ipv4", + requestFamily: v1.IPv4Protocol, + expectedResult: "", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIP: "2000::1", + }, + }, + }, + + { + name: "service single stack ipv4. want ipv4", + requestFamily: v1.IPv4Protocol, + expectedResult: "10.0.0.10", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"10.0.0.10"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + }, + + { + name: "service single stack ipv4. want ipv6", + requestFamily: v1.IPv6Protocol, + expectedResult: "", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"10.0.0.10"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + }, + + { + name: "service single stack ipv6. want ipv6", + requestFamily: v1.IPv6Protocol, + expectedResult: "2000::1", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"2000::1"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, + }, + }, + + { + name: "service single stack ipv6. want ipv4", + requestFamily: v1.IPv4Protocol, + expectedResult: "", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"2000::1"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, + }, + }, + // dual stack + { + name: "service dual stack ipv4,6. want ipv4", + requestFamily: v1.IPv4Protocol, + expectedResult: "10.0.0.10", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, + }, + }, + + { + name: "service dual stack ipv4,6. want ipv6", + requestFamily: v1.IPv6Protocol, + expectedResult: "2000::1", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, + }, + }, + + { + name: "service dual stack ipv6,4. want ipv6", + requestFamily: v1.IPv6Protocol, + expectedResult: "2000::1", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"2000::1", "10.0.0.10"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + }, + }, + }, + + { + name: "service dual stack ipv6,4. want ipv4", + requestFamily: v1.IPv4Protocol, + expectedResult: "10.0.0.10", + service: v1.Service{ + Spec: v1.ServiceSpec{ + ClusterIPs: []string{"2000::1", "10.0.0.10"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + ip := GetClusterIPByFamily(testCase.requestFamily, &testCase.service) + if ip != testCase.expectedResult { + t.Fatalf("expected ip:%v got %v", testCase.expectedResult, ip) + } + }) + } + +} diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index 4718639c14e..64dc752e93c 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -604,8 +604,12 @@ func NewProxier( isIPv6Mode: isIPv6, } - serviceChanges := proxy.NewServiceChangeTracker(proxier.newServiceInfo, &isIPv6, recorder, proxier.serviceMapChange) - endPointChangeTracker := proxy.NewEndpointChangeTracker(hostname, proxier.newEndpointInfo, &isIPv6, recorder, endpointSlicesEnabled, proxier.endpointsMapChange) + ipFamily := v1.IPv4Protocol + if isIPv6 { + ipFamily = v1.IPv6Protocol + } + serviceChanges := proxy.NewServiceChangeTracker(proxier.newServiceInfo, ipFamily, recorder, proxier.serviceMapChange) + endPointChangeTracker := proxy.NewEndpointChangeTracker(hostname, proxier.newEndpointInfo, ipFamily, recorder, endpointSlicesEnabled, proxier.endpointsMapChange) proxier.endpointsChanges = endPointChangeTracker proxier.serviceChanges = serviceChanges diff --git a/pkg/proxy/winkernel/proxier_test.go b/pkg/proxy/winkernel/proxier_test.go index f3a92c21716..a24c49fb07b 100644 --- a/pkg/proxy/winkernel/proxier_test.go +++ b/pkg/proxy/winkernel/proxier_test.go @@ -124,9 +124,8 @@ func NewFakeProxier(syncPeriod time.Duration, minSyncPeriod time.Duration, clust endPointsRefCount: make(endPointsReferenceCountMap), } - isIPv6 := false - serviceChanges := proxy.NewServiceChangeTracker(proxier.newServiceInfo, &isIPv6, nil, proxier.serviceMapChange) - endpointChangeTracker := proxy.NewEndpointChangeTracker(hostname, proxier.newEndpointInfo, &isIPv6, nil, endpointSliceEnabled, proxier.endpointsMapChange) + serviceChanges := proxy.NewServiceChangeTracker(proxier.newServiceInfo, v1.IPv4Protocol, nil, proxier.serviceMapChange) + endpointChangeTracker := proxy.NewEndpointChangeTracker(hostname, proxier.newEndpointInfo, v1.IPv4Protocol, nil, endpointSliceEnabled, proxier.endpointsMapChange) proxier.endpointsChanges = endpointChangeTracker proxier.serviceChanges = serviceChanges diff --git a/pkg/registry/core/service/ipallocator/controller/BUILD b/pkg/registry/core/service/ipallocator/controller/BUILD index 959f42ba5f6..ad8f45f1e44 100644 --- a/pkg/registry/core/service/ipallocator/controller/BUILD +++ b/pkg/registry/core/service/ipallocator/controller/BUILD @@ -14,7 +14,6 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1/helper:go_default_library", - "//pkg/features:go_default_library", "//pkg/registry/core/rangeallocation:go_default_library", "//pkg/registry/core/service/ipallocator:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -22,7 +21,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", diff --git a/pkg/registry/core/service/ipallocator/controller/repair.go b/pkg/registry/core/service/ipallocator/controller/repair.go index ce067f7e2c0..97447dcc163 100644 --- a/pkg/registry/core/service/ipallocator/controller/repair.go +++ b/pkg/registry/core/service/ipallocator/controller/repair.go @@ -36,9 +36,6 @@ import ( "k8s.io/kubernetes/pkg/registry/core/rangeallocation" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" netutil "k8s.io/utils/net" - - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/features" ) // Repair is a controller loop that periodically examines all service ClusterIP allocations @@ -60,13 +57,11 @@ type Repair struct { interval time.Duration serviceClient corev1client.ServicesGetter - network *net.IPNet - alloc rangeallocation.RangeRegistry - secondaryNetwork *net.IPNet - secondaryAlloc rangeallocation.RangeRegistry + networkByFamily map[v1.IPFamily]*net.IPNet // networks we operate on, by their family + allocatorByFamily map[v1.IPFamily]rangeallocation.RangeRegistry // allocators we use, by their family - leaks map[string]int // counter per leaked IP - recorder record.EventRecorder + leaksByFamily map[v1.IPFamily]map[string]int // counter per leaked IP per family + recorder record.EventRecorder } // How many times we need to detect a leak before we clean up. This is to @@ -80,17 +75,39 @@ func NewRepair(interval time.Duration, serviceClient corev1client.ServicesGetter eventBroadcaster.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: eventClient.Events("")}) recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "ipallocator-repair-controller"}) + // build *ByFamily struct members + networkByFamily := make(map[v1.IPFamily]*net.IPNet) + allocatorByFamily := make(map[v1.IPFamily]rangeallocation.RangeRegistry) + leaksByFamily := make(map[v1.IPFamily]map[string]int) + + primary := v1.IPv4Protocol + secondary := v1.IPv6Protocol + if netutil.IsIPv6(network.IP) { + primary = v1.IPv6Protocol + } + + networkByFamily[primary] = network + allocatorByFamily[primary] = alloc + leaksByFamily[primary] = make(map[string]int) + + if secondaryNetwork != nil && secondaryNetwork.IP != nil { + if primary == v1.IPv6Protocol { + secondary = v1.IPv4Protocol + } + networkByFamily[secondary] = secondaryNetwork + allocatorByFamily[secondary] = secondaryAlloc + leaksByFamily[secondary] = make(map[string]int) + } + return &Repair{ interval: interval, serviceClient: serviceClient, - network: network, - alloc: alloc, - secondaryNetwork: secondaryNetwork, - secondaryAlloc: secondaryAlloc, + networkByFamily: networkByFamily, + allocatorByFamily: allocatorByFamily, - leaks: map[string]int{}, - recorder: recorder, + leaksByFamily: leaksByFamily, + recorder: recorder, } } @@ -108,29 +125,6 @@ func (c *Repair) RunOnce() error { return retry.RetryOnConflict(retry.DefaultBackoff, c.runOnce) } -// selectAllocForIP returns an allocator for an IP based weather it belongs to the primary or the secondary allocator -func (c *Repair) selectAllocForIP(ip net.IP, primary ipallocator.Interface, secondary ipallocator.Interface) ipallocator.Interface { - if !c.shouldWorkOnSecondary() { - return primary - } - - cidr := secondary.CIDR() - if netutil.IsIPv6CIDR(&cidr) && netutil.IsIPv6(ip) { - return secondary - } - - return primary -} - -// shouldWorkOnSecondary returns true if the repairer should perform work for secondary network (dual stack) -func (c *Repair) shouldWorkOnSecondary() bool { - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - return false - } - - return c.secondaryNetwork != nil && c.secondaryNetwork.IP != nil -} - // runOnce verifies the state of the cluster IP allocations and returns an error if an unrecoverable problem occurs. func (c *Repair) runOnce() error { // TODO: (per smarterclayton) if Get() or ListServices() is a weak consistency read, @@ -142,50 +136,55 @@ func (c *Repair) runOnce() error { // If etcd server is not running we should wait for some time and fail only then. This is particularly // important when we start apiserver and etcd at the same time. - var snapshot *api.RangeAllocation - var secondarySnapshot *api.RangeAllocation + snapshotByFamily := make(map[v1.IPFamily]*api.RangeAllocation) + storedByFamily := make(map[v1.IPFamily]ipallocator.Interface) - var stored, secondaryStored ipallocator.Interface - var err, secondaryErr error + err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + for family, allocator := range c.allocatorByFamily { + // get snapshot if it is not there + if _, ok := snapshotByFamily[family]; !ok { + snapshot, err := allocator.Get() + if err != nil { + return false, err + } - err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - var err error - snapshot, err = c.alloc.Get() - if err != nil { - return false, err - } - - if c.shouldWorkOnSecondary() { - secondarySnapshot, err = c.secondaryAlloc.Get() - if err != nil { - return false, err + snapshotByFamily[family] = snapshot } } - return true, nil }) + if err != nil { return fmt.Errorf("unable to refresh the service IP block: %v", err) } - // If not yet initialized. - if snapshot.Range == "" { - snapshot.Range = c.network.String() + + // ensure that ranges are assigned + for family, snapshot := range snapshotByFamily { + if snapshot.Range == "" { + snapshot.Range = c.networkByFamily[family].String() + } } - if c.shouldWorkOnSecondary() && secondarySnapshot.Range == "" { - secondarySnapshot.Range = c.secondaryNetwork.String() - } // Create an allocator because it is easy to use. + for family, snapshot := range snapshotByFamily { + stored, err := ipallocator.NewFromSnapshot(snapshot) + if err != nil { + return fmt.Errorf("unable to rebuild allocator from snapshots for family:%v with error:%v", family, err) + } - stored, err = ipallocator.NewFromSnapshot(snapshot) - if c.shouldWorkOnSecondary() { - secondaryStored, secondaryErr = ipallocator.NewFromSnapshot(secondarySnapshot) + storedByFamily[family] = stored } - if err != nil || secondaryErr != nil { - return fmt.Errorf("unable to rebuild allocator from snapshots: %v", err) - } + rebuiltByFamily := make(map[v1.IPFamily]*ipallocator.Range) + for family, network := range c.networkByFamily { + rebuilt, err := ipallocator.NewCIDRRange(network) + if err != nil { + return fmt.Errorf("unable to create CIDR range for family %v: %v", family, err) + } + + rebuiltByFamily[family] = rebuilt + } // We explicitly send no resource version, since the resource version // of 'snapshot' is from a different collection, it's not comparable to // the service collection. The caching layer keeps per-collection RVs, @@ -196,18 +195,11 @@ func (c *Repair) runOnce() error { return fmt.Errorf("unable to refresh the service IP block: %v", err) } - var rebuilt, secondaryRebuilt *ipallocator.Range - rebuilt, err = ipallocator.NewCIDRRange(c.network) - if err != nil { - return fmt.Errorf("unable to create CIDR range: %v", err) - } - - if c.shouldWorkOnSecondary() { - secondaryRebuilt, err = ipallocator.NewCIDRRange(c.secondaryNetwork) - } - - if err != nil { - return fmt.Errorf("unable to create CIDR range: %v", err) + getFamilyByIP := func(ip net.IP) v1.IPFamily { + if netutil.IsIPv6(ip) { + return v1.IPv6Protocol + } + return v1.IPv4Protocol } // Check every Service's ClusterIP, and rebuild the state as we think it should be. @@ -216,64 +208,72 @@ func (c *Repair) runOnce() error { // didn't need a cluster IP continue } - ip := net.ParseIP(svc.Spec.ClusterIP) - if ip == nil { - // cluster IP is corrupt - c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPNotValid", "Cluster IP %s is not a valid IP; please recreate service", svc.Spec.ClusterIP) - runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not a valid IP; please recreate", svc.Spec.ClusterIP, svc.Name, svc.Namespace)) - continue - } - // mark it as in-use - actualAlloc := c.selectAllocForIP(ip, rebuilt, secondaryRebuilt) - switch err := actualAlloc.Allocate(ip); err { - case nil: - actualStored := c.selectAllocForIP(ip, stored, secondaryStored) - if actualStored.Has(ip) { - // remove it from the old set, so we can find leaks - actualStored.Release(ip) - } else { - // cluster IP doesn't seem to be allocated - c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPNotAllocated", "Cluster IP %s is not allocated; repairing", ip) - runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not allocated; repairing", ip, svc.Name, svc.Namespace)) + for _, ip := range svc.Spec.ClusterIPs { + ip := net.ParseIP(ip) + if ip == nil { + // cluster IP is corrupt + c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPNotValid", "Cluster IP %s is not a valid IP; please recreate service", ip) + runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not a valid IP; please recreate", ip, svc.Name, svc.Namespace)) + continue + } + + family := getFamilyByIP(ip) + if _, ok := rebuiltByFamily[family]; !ok { + // this service is using an IPFamily no longer configured on cluster + c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPNotValid", "Cluster IP %s(%s) is of ip family that is no longer configured on cluster; please recreate service", ip, family) + runtime.HandleError(fmt.Errorf("the cluster IP %s(%s) for service %s/%s is of ip family that is no longer configured on cluster; please recreate", ip, family, svc.Name, svc.Namespace)) + continue + } + + // mark it as in-use + actualAlloc := rebuiltByFamily[family] + switch err := actualAlloc.Allocate(ip); err { + case nil: + actualStored := storedByFamily[family] + if actualStored.Has(ip) { + // remove it from the old set, so we can find leaks + actualStored.Release(ip) + } else { + // cluster IP doesn't seem to be allocated + c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPNotAllocated", "Cluster IP [%v]:%s is not allocated; repairing", family, ip) + runtime.HandleError(fmt.Errorf("the cluster IP [%v]:%s for service %s/%s is not allocated; repairing", family, ip, svc.Name, svc.Namespace)) + } + delete(c.leaksByFamily[family], ip.String()) // it is used, so it can't be leaked + case ipallocator.ErrAllocated: + // cluster IP is duplicate + c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPAlreadyAllocated", "Cluster IP [%v]:%s was assigned to multiple services; please recreate service", family, ip) + runtime.HandleError(fmt.Errorf("the cluster IP [%v]:%s for service %s/%s was assigned to multiple services; please recreate", family, ip, svc.Name, svc.Namespace)) + case err.(*ipallocator.ErrNotInRange): + // cluster IP is out of range + c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPOutOfRange", "Cluster IP [%v]:%s is not within the service CIDR %s; please recreate service", family, ip, c.networkByFamily[family]) + runtime.HandleError(fmt.Errorf("the cluster IP [%v]:%s for service %s/%s is not within the service CIDR %s; please recreate", family, ip, svc.Name, svc.Namespace, c.networkByFamily[family])) + case ipallocator.ErrFull: + // somehow we are out of IPs + cidr := actualAlloc.CIDR() + c.recorder.Eventf(&svc, v1.EventTypeWarning, "ServiceCIDRFull", "Service CIDR %v is full; you must widen the CIDR in order to create new services for Cluster IP [%v]:%s", cidr, family, ip) + return fmt.Errorf("the service CIDR %v is full; you must widen the CIDR in order to create new services for Cluster IP [%v]:%s", cidr, family, ip) + default: + c.recorder.Eventf(&svc, v1.EventTypeWarning, "UnknownError", "Unable to allocate cluster IP [%v]:%s due to an unknown error", family, ip) + return fmt.Errorf("unable to allocate cluster IP [%v]:%s for service %s/%s due to an unknown error, exiting: %v", family, ip, svc.Name, svc.Namespace, err) } - delete(c.leaks, ip.String()) // it is used, so it can't be leaked - case ipallocator.ErrAllocated: - // cluster IP is duplicate - c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPAlreadyAllocated", "Cluster IP %s was assigned to multiple services; please recreate service", ip) - runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s was assigned to multiple services; please recreate", ip, svc.Name, svc.Namespace)) - case err.(*ipallocator.ErrNotInRange): - // cluster IP is out of range - c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPOutOfRange", "Cluster IP %s is not within the service CIDR %s; please recreate service", ip, c.network) - runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not within the service CIDR %s; please recreate", ip, svc.Name, svc.Namespace, c.network)) - case ipallocator.ErrFull: - // somehow we are out of IPs - cidr := actualAlloc.CIDR() - c.recorder.Eventf(&svc, v1.EventTypeWarning, "ServiceCIDRFull", "Service CIDR %v is full; you must widen the CIDR in order to create new services", cidr) - return fmt.Errorf("the service CIDR %v is full; you must widen the CIDR in order to create new services", cidr) - default: - c.recorder.Eventf(&svc, v1.EventTypeWarning, "UnknownError", "Unable to allocate cluster IP %s due to an unknown error", ip) - return fmt.Errorf("unable to allocate cluster IP %s for service %s/%s due to an unknown error, exiting: %v", ip, svc.Name, svc.Namespace, err) } } - c.checkLeaked(stored, rebuilt) - if c.shouldWorkOnSecondary() { - c.checkLeaked(secondaryStored, secondaryRebuilt) + // leak check + for family, leaks := range c.leaksByFamily { + c.checkLeaked(leaks, storedByFamily[family], rebuiltByFamily[family]) } + // save logic // Blast the rebuilt state into storage. - err = c.saveSnapShot(rebuilt, c.alloc, snapshot) - if err != nil { - return err - } - - if c.shouldWorkOnSecondary() { - err := c.saveSnapShot(secondaryRebuilt, c.secondaryAlloc, secondarySnapshot) + for family, rebuilt := range rebuiltByFamily { + err = c.saveSnapShot(rebuilt, c.allocatorByFamily[family], snapshotByFamily[family]) if err != nil { - return nil + return err } } + return nil } @@ -291,10 +291,10 @@ func (c *Repair) saveSnapShot(rebuilt *ipallocator.Range, alloc rangeallocation. return nil } -func (c *Repair) checkLeaked(stored ipallocator.Interface, rebuilt *ipallocator.Range) { +func (c *Repair) checkLeaked(leaks map[string]int, stored ipallocator.Interface, rebuilt *ipallocator.Range) { // Check for IPs that are left in the old set. They appear to have been leaked. stored.ForEach(func(ip net.IP) { - count, found := c.leaks[ip.String()] + count, found := leaks[ip.String()] switch { case !found: // flag it to be cleaned up after any races (hopefully) are gone @@ -303,7 +303,7 @@ func (c *Repair) checkLeaked(stored ipallocator.Interface, rebuilt *ipallocator. fallthrough case count > 0: // pretend it is still in use until count expires - c.leaks[ip.String()] = count - 1 + leaks[ip.String()] = count - 1 if err := rebuilt.Allocate(ip); err != nil { runtime.HandleError(fmt.Errorf("the cluster IP %s may have leaked, but can not be allocated: %v", ip, err)) } @@ -312,5 +312,4 @@ func (c *Repair) checkLeaked(stored ipallocator.Interface, rebuilt *ipallocator. runtime.HandleError(fmt.Errorf("the cluster IP %s appears to have leaked: cleaning up", ip)) } }) - } diff --git a/pkg/registry/core/service/ipallocator/controller/repair_test.go b/pkg/registry/core/service/ipallocator/controller/repair_test.go index 9b83282e5c5..5f41441fc9a 100644 --- a/pkg/registry/core/service/ipallocator/controller/repair_test.go +++ b/pkg/registry/core/service/ipallocator/controller/repair_test.go @@ -23,6 +23,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" api "k8s.io/kubernetes/pkg/apis/core" @@ -147,27 +148,49 @@ func TestRepairWithExisting(t *testing.T) { fakeClient := fake.NewSimpleClientset( &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "one"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.1"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, }, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Namespace: "two", Name: "two"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.100"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.1.100", + ClusterIPs: []string{"192.168.1.100"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, }, &corev1.Service{ // outside CIDR, will be dropped ObjectMeta: metav1.ObjectMeta{Namespace: "three", Name: "three"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.0.1"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.0.1", + ClusterIPs: []string{"192.168.0.1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, }, &corev1.Service{ // empty, ignored ObjectMeta: metav1.ObjectMeta{Namespace: "four", Name: "four"}, - Spec: corev1.ServiceSpec{ClusterIP: ""}, + Spec: corev1.ServiceSpec{ + ClusterIP: "", + ClusterIPs: []string{""}, + }, }, &corev1.Service{ // duplicate, dropped ObjectMeta: metav1.ObjectMeta{Namespace: "five", Name: "five"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.1"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, }, &corev1.Service{ // headless ObjectMeta: metav1.ObjectMeta{Namespace: "six", Name: "six"}, - Spec: corev1.ServiceSpec{ClusterIP: "None"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + ClusterIPs: []string{"None"}, + }, }, ) @@ -192,7 +215,7 @@ func TestRepairWithExisting(t *testing.T) { t.Errorf("unexpected ipallocator state: %#v", after) } if free := after.Free(); free != 252 { - t.Errorf("unexpected ipallocator state: %d free", free) + t.Errorf("unexpected ipallocator state: %d free (expected 252)", free) } } @@ -229,44 +252,38 @@ func makeIPNet(cidr string) *net.IPNet { } func TestShouldWorkOnSecondary(t *testing.T) { testCases := []struct { - name string - enableDualStack bool - expectedResult bool - primaryNet *net.IPNet - secondaryNet *net.IPNet + name string + expectedFamilies []v1.IPFamily + primaryNet *net.IPNet + secondaryNet *net.IPNet }{ { - name: "not a dual stack, primary only", - enableDualStack: false, - expectedResult: false, - primaryNet: makeIPNet("10.0.0.0/16"), - secondaryNet: nil, + name: "primary only (v4)", + expectedFamilies: []v1.IPFamily{v1.IPv4Protocol}, + primaryNet: makeIPNet("10.0.0.0/16"), + secondaryNet: nil, }, { - name: "not a dual stack, primary and secondary provided", - enableDualStack: false, - expectedResult: false, - primaryNet: makeIPNet("10.0.0.0/16"), - secondaryNet: makeIPNet("2000::/120"), + name: "primary only (v6)", + expectedFamilies: []v1.IPFamily{v1.IPv6Protocol}, + primaryNet: makeIPNet("2000::/120"), + secondaryNet: nil, }, { - name: "dual stack, primary only", - enableDualStack: true, - expectedResult: false, - primaryNet: makeIPNet("10.0.0.0/16"), - secondaryNet: nil, + name: "primary and secondary provided (v4,v6)", + expectedFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + primaryNet: makeIPNet("10.0.0.0/16"), + secondaryNet: makeIPNet("2000::/120"), }, { - name: "dual stack, primary and secondary", - enableDualStack: true, - expectedResult: true, - primaryNet: makeIPNet("10.0.0.0/16"), - secondaryNet: makeIPNet("2000::/120"), + name: "primary and secondary provided (v6,v4)", + expectedFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + primaryNet: makeIPNet("2000::/120"), + secondaryNet: makeIPNet("10.0.0.0/16"), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() fakeClient := makeFakeClientSet() primaryRegistry := makeRangeRegistry(t, tc.primaryNet.String()) @@ -277,8 +294,33 @@ func TestShouldWorkOnSecondary(t *testing.T) { } repair := NewRepair(0, fakeClient.CoreV1(), fakeClient.CoreV1(), tc.primaryNet, primaryRegistry, tc.secondaryNet, secondaryRegistry) - if repair.shouldWorkOnSecondary() != tc.expectedResult { - t.Errorf("shouldWorkOnSecondary should be %v and found %v", tc.expectedResult, repair.shouldWorkOnSecondary()) + if len(repair.allocatorByFamily) != len(tc.expectedFamilies) { + t.Fatalf("expected to have allocator by family count:%v got %v", len(tc.expectedFamilies), len(repair.allocatorByFamily)) + } + + seen := make(map[v1.IPFamily]bool) + for _, family := range tc.expectedFamilies { + familySeen := true + + if _, ok := repair.allocatorByFamily[family]; !ok { + familySeen = familySeen && ok + } + + if _, ok := repair.networkByFamily[family]; !ok { + familySeen = familySeen && ok + } + + if _, ok := repair.leaksByFamily[family]; !ok { + familySeen = familySeen && ok + } + + seen[family] = familySeen + } + + for family, seen := range seen { + if !seen { + t.Fatalf("expected repair look to have family %v, but it was not visible on either (or all) network, allocator, leaks", family) + } } }) } @@ -418,6 +460,11 @@ func TestRepairLeakDualStack(t *testing.T) { } func TestRepairWithExistingDualStack(t *testing.T) { + // because anything (other than allocator) depends + // on families assigned to service (not the value of IPFamilyPolicy) + // we can saftly create tests that has ipFamilyPolicy:nil + // this will work every where except alloc & validation + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() _, cidr, _ := net.ParseCIDR("192.168.1.0/24") previous, err := ipallocator.NewCIDRRange(cidr) @@ -445,44 +492,94 @@ func TestRepairWithExistingDualStack(t *testing.T) { fakeClient := fake.NewSimpleClientset( &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "one"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.1"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x1", Name: "one-v4-v6"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1", "2000::1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, }, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "one-v6"}, - Spec: corev1.ServiceSpec{ClusterIP: "2000::1"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x2", Name: "one-v6-v4"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1", "192.168.1.100"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + }, }, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Namespace: "two", Name: "two"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.100"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x3", Name: "two-6"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "2000::2", + ClusterIPs: []string{"2000::2"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, }, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Namespace: "two", Name: "two-6"}, - Spec: corev1.ServiceSpec{ClusterIP: "2000::2"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x4", Name: "two-4"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.1.90", + ClusterIPs: []string{"192.168.1.90"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + // outside CIDR, will be dropped + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "x5", Name: "out-v4"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.0.1", + ClusterIPs: []string{"192.168.0.1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, }, &corev1.Service{ // outside CIDR, will be dropped - ObjectMeta: metav1.ObjectMeta{Namespace: "three", Name: "three"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.0.1"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x6", Name: "out-v6"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "3000::1", + ClusterIPs: []string{"3000::1"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, }, - &corev1.Service{ // outside CIDR, will be dropped - ObjectMeta: metav1.ObjectMeta{Namespace: "three", Name: "three-v6"}, - Spec: corev1.ServiceSpec{ClusterIP: "3000::1"}, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "x6", Name: "out-v4-v6"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.0.1", + ClusterIPs: []string{"192.168.0.1", "3000::1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }, }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "x6", Name: "out-v6-v4"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "3000::1", + ClusterIPs: []string{"3000::1", "192.168.0.1"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + }, + }, + &corev1.Service{ // empty, ignored - ObjectMeta: metav1.ObjectMeta{Namespace: "four", Name: "four"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x7", Name: "out-empty"}, Spec: corev1.ServiceSpec{ClusterIP: ""}, }, &corev1.Service{ // duplicate, dropped - ObjectMeta: metav1.ObjectMeta{Namespace: "five", Name: "five"}, - Spec: corev1.ServiceSpec{ClusterIP: "192.168.1.1"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x8", Name: "duplicate"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "192.168.1.1", + ClusterIPs: []string{"192.168.1.1"}, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, }, &corev1.Service{ // duplicate, dropped - ObjectMeta: metav1.ObjectMeta{Namespace: "five", Name: "five-v6"}, - Spec: corev1.ServiceSpec{ClusterIP: "2000::2"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x9", Name: "duplicate-v6"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "2000::2", + ClusterIPs: []string{"2000::2"}, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }, }, &corev1.Service{ // headless - ObjectMeta: metav1.ObjectMeta{Namespace: "six", Name: "six"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "x10", Name: "headless"}, Spec: corev1.ServiceSpec{ClusterIP: "None"}, }, ) @@ -519,9 +616,10 @@ func TestRepairWithExistingDualStack(t *testing.T) { if !after.Has(net.ParseIP("192.168.1.1")) || !after.Has(net.ParseIP("192.168.1.100")) { t.Errorf("unexpected ipallocator state: %#v", after) } - if free := after.Free(); free != 252 { - t.Errorf("unexpected ipallocator state: %d free (number of free ips is not 252)", free) + if free := after.Free(); free != 251 { + t.Errorf("unexpected ipallocator state: %d free (number of free ips is not 251)", free) } + secondaryAfter, err := ipallocator.NewFromSnapshot(secondaryIPRegistry.updated) if err != nil { t.Fatal(err) @@ -532,5 +630,4 @@ func TestRepairWithExistingDualStack(t *testing.T) { if free := secondaryAfter.Free(); free != 65533 { t.Errorf("unexpected ipallocator state: %d free (number of free ips is not 65532)", free) } - } diff --git a/pkg/registry/core/service/storage/BUILD b/pkg/registry/core/service/storage/BUILD index 69c036254e5..7c98400738f 100644 --- a/pkg/registry/core/service/storage/BUILD +++ b/pkg/registry/core/service/storage/BUILD @@ -29,7 +29,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", @@ -41,6 +40,7 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/util/dryrun:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) @@ -54,7 +54,6 @@ go_library( deps = [ "//pkg/api/service:go_default_library", "//pkg/apis/core:go_default_library", - "//pkg/apis/core/helper:go_default_library", "//pkg/apis/core/validation:go_default_library", "//pkg/features:go_default_library", "//pkg/printers:go_default_library", diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go index 0d5379f8c11..af759b5dcc3 100644 --- a/pkg/registry/core/service/storage/rest.go +++ b/pkg/registry/core/service/storage/rest.go @@ -40,7 +40,6 @@ import ( apiservice "k8s.io/kubernetes/pkg/api/service" api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/apis/core/helper" "k8s.io/kubernetes/pkg/apis/core/validation" registry "k8s.io/kubernetes/pkg/registry/core/service" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" @@ -53,14 +52,14 @@ import ( // REST adapts a service registry into apiserver's RESTStorage model. type REST struct { - strategy rest.RESTCreateUpdateStrategy - services ServiceStorage - endpoints EndpointsStorage - serviceIPs ipallocator.Interface - secondaryServiceIPs ipallocator.Interface - serviceNodePorts portallocator.Interface - proxyTransport http.RoundTripper - pods rest.Getter + strategy rest.RESTCreateUpdateStrategy + services ServiceStorage + endpoints EndpointsStorage + serviceIPAllocatorsByFamily map[api.IPFamily]ipallocator.Interface + defaultServiceIPFamily api.IPFamily // --service-cluster-ip-range[0] + serviceNodePorts portallocator.Interface + proxyTransport http.RoundTripper + pods rest.Getter } // ServiceNodePort includes protocol and port number of a service NodePort. @@ -105,15 +104,41 @@ func NewREST( strategy, _ := registry.StrategyForServiceCIDRs(serviceIPs.CIDR(), secondaryServiceIPs != nil) + byIPFamily := make(map[api.IPFamily]ipallocator.Interface) + + // detect this cluster default Service IPFamily (ipfamily of --service-cluster-ip-range[0]) + serviceIPFamily := api.IPv4Protocol + cidr := serviceIPs.CIDR() + if netutil.IsIPv6CIDR(&cidr) { + serviceIPFamily = api.IPv6Protocol + } + + // add primary family + byIPFamily[serviceIPFamily] = serviceIPs + + if secondaryServiceIPs != nil { + // process secondary family + secondaryServiceIPFamily := api.IPv6Protocol + + // get family of secondary + if serviceIPFamily == api.IPv6Protocol { + secondaryServiceIPFamily = api.IPv4Protocol + } + // add it + byIPFamily[secondaryServiceIPFamily] = secondaryServiceIPs + } + + klog.V(0).Infof("the default service ipfamily for this cluster is: %s", string(serviceIPFamily)) + rest := &REST{ - strategy: strategy, - services: services, - endpoints: endpoints, - serviceIPs: serviceIPs, - secondaryServiceIPs: secondaryServiceIPs, - serviceNodePorts: serviceNodePorts, - proxyTransport: proxyTransport, - pods: pods, + strategy: strategy, + services: services, + endpoints: endpoints, + serviceIPAllocatorsByFamily: byIPFamily, + serviceNodePorts: serviceNodePorts, + defaultServiceIPFamily: serviceIPFamily, + proxyTransport: proxyTransport, + pods: pods, } return rest, ®istry.ProxyREST{Redirector: rest, ProxyTransport: proxyTransport} @@ -171,28 +196,35 @@ func (rs *REST) Export(ctx context.Context, name string, opts metav1.ExportOptio func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { service := obj.(*api.Service) + // bag of clusterIPs allocated in the process of creation + // failed allocation will automatically trigger release + var toReleaseClusterIPs map[api.IPFamily]string + if err := rest.BeforeCreate(rs.strategy, ctx, obj); err != nil { return nil, err } // TODO: this should probably move to strategy.PrepareForCreate() - releaseServiceIP := false defer func() { - if releaseServiceIP { - if helper.IsServiceIPSet(service) { - allocator := rs.getAllocatorByClusterIP(service) - allocator.Release(net.ParseIP(service.Spec.ClusterIP)) - } + released, err := rs.releaseClusterIPs(toReleaseClusterIPs) + if err != nil { + klog.Warningf("failed to release clusterIPs for failed new service:%v allocated:%v released:%v error:%v", + service.Name, toReleaseClusterIPs, released, err) } }() + // try set ip families (for missing ip families) + // we do it here, since we want this to be visible + // even when dryRun == true + if err := rs.tryDefaultValidateServiceClusterIPFields(service); err != nil { + return nil, err + } + var err error if !dryrun.IsDryRun(options.DryRun) { - if service.Spec.Type != api.ServiceTypeExternalName { - allocator := rs.getAllocatorBySpec(service) - if releaseServiceIP, err = initClusterIP(service, allocator); err != nil { - return nil, err - } + toReleaseClusterIPs, err = rs.allocServiceClusterIPs(service) + if err != nil { + return nil, err } } @@ -227,7 +259,8 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation utilruntime.HandleError(fmt.Errorf("error(s) committing service node-ports changes: %v", el)) } - releaseServiceIP = false + // no clusterips to release + toReleaseClusterIPs = nil } return out, err @@ -241,6 +274,18 @@ func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.Val } svc := obj.(*api.Service) + // (khenidak) double check that this is in fact the best place for this + + // delete strategy handles graceful delete only. It expects strategy + // to implement Graceful-Delete related interface. Hence we are not doing + // the below there. instead we are doing it locally. Until strategy.BeforeDelete works without + // having to implement graceful delete management + // set ClusterIPs based on ClusterIP + // because we depend on ClusterIPs and data might be saved without ClusterIPs .. + + if svc.Spec.ClusterIPs == nil && len(svc.Spec.ClusterIP) > 0 { + svc.Spec.ClusterIPs = []string{svc.Spec.ClusterIP} + } // Only perform the cleanup if this is a non-dryrun deletion if !dryrun.IsDryRun(options.DryRun) { @@ -268,10 +313,7 @@ func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.Val } func (rs *REST) releaseAllocatedResources(svc *api.Service) { - if helper.IsServiceIPSet(svc) { - allocator := rs.getAllocatorByClusterIP(svc) - allocator.Release(net.ParseIP(svc.Spec.ClusterIP)) - } + rs.releaseServiceClusterIPs(svc) for _, nodePort := range collectServiceNodePorts(svc) { err := rs.serviceNodePorts.Release(nodePort) @@ -372,7 +414,6 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj return nil, false, err } oldService := oldObj.(*api.Service) - obj, err := objInfo.UpdatedObject(ctx, oldService) if err != nil { return nil, false, err @@ -389,13 +430,24 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj return nil, false, err } - // TODO: this should probably move to strategy.PrepareForCreate() - releaseServiceIP := false + var allocated map[api.IPFamily]string + var toReleaseIPs map[api.IPFamily]string + + performRelease := false // when set, any clusterIP that should be released will be released + // cleanup + // on failure: Any allocated ip must be released back + // on failure: any ip that should be released, will *not* be released + // on success: any ip that should be released, will be released defer func() { - if releaseServiceIP { - if helper.IsServiceIPSet(service) { - allocator := rs.getAllocatorByClusterIP(service) - allocator.Release(net.ParseIP(service.Spec.ClusterIP)) + // release the allocated, this is expected to be cleared if the entire function ran to success + if allocated_released, err := rs.releaseClusterIPs(allocated); err != nil { + klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. Allocated/Released:%v/%v", service.Namespace, service.Name, err, allocated, allocated_released) + + } + // performRelease is set when the enture function ran to success + if performRelease { + if toReleaseIPs_released, err := rs.releaseClusterIPs(toReleaseIPs); err != nil { + klog.V(4).Infof("service %v/%v failed to clean up after failed service update error:%v. ShouldRelease/Released:%v/%v", service.Namespace, service.Name, err, toReleaseIPs, toReleaseIPs_released) } } }() @@ -403,22 +455,15 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj nodePortOp := portallocator.StartOperation(rs.serviceNodePorts, dryrun.IsDryRun(options.DryRun)) defer nodePortOp.Finish() + // try set ip families (for missing ip families) + if err := rs.tryDefaultValidateServiceClusterIPFields(service); err != nil { + return nil, false, err + } + if !dryrun.IsDryRun(options.DryRun) { - // Update service from ExternalName to non-ExternalName, should initialize ClusterIP. - // Since we don't support changing the ip family of a service we don't need to handle - // oldService.Spec.ServiceIPFamily != service.Spec.ServiceIPFamily - if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName { - allocator := rs.getAllocatorBySpec(service) - if releaseServiceIP, err = initClusterIP(service, allocator); err != nil { - return nil, false, err - } - } - // Update service from non-ExternalName to ExternalName, should release ClusterIP if exists. - if oldService.Spec.Type != api.ServiceTypeExternalName && service.Spec.Type == api.ServiceTypeExternalName { - if helper.IsServiceIPSet(oldService) { - allocator := rs.getAllocatorByClusterIP(service) - allocator.Release(net.ParseIP(oldService.Spec.ClusterIP)) - } + allocated, toReleaseIPs, err = rs.handleClusterIPsForUpdatedService(oldService, service) + if err != nil { + return nil, false, err } } // Update service from NodePort or LoadBalancer to ExternalName or ClusterIP, should release NodePort if exists. @@ -455,9 +500,10 @@ func (rs *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObj // problems should be fixed by an eventual reconciliation / restart utilruntime.HandleError(fmt.Errorf("error(s) committing NodePorts changes: %v", el)) } - - releaseServiceIP = false } + // all good + allocated = nil // if something was allocated, keep it allocated + performRelease = true // if something that should be released then go ahead and release it return out, created, err } @@ -541,33 +587,397 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO return r.services.ConvertToTable(ctx, object, tableOptions) } -// When allocating we always use BySpec, when releasing we always use ByClusterIP -func (r *REST) getAllocatorByClusterIP(service *api.Service) ipallocator.Interface { - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || r.secondaryServiceIPs == nil { - return r.serviceIPs - } +func (rs *REST) allocClusterIPs(service *api.Service, toAlloc map[api.IPFamily]string) (map[api.IPFamily]string, error) { + allocated := make(map[api.IPFamily]string) - secondaryAllocatorCIDR := r.secondaryServiceIPs.CIDR() - if netutil.IsIPv6String(service.Spec.ClusterIP) == netutil.IsIPv6CIDR(&secondaryAllocatorCIDR) { - return r.secondaryServiceIPs + for family, ip := range toAlloc { + allocator := rs.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate + if ip == "" { + allocatedIP, err := allocator.AllocateNext() + if err != nil { + return allocated, errors.NewInternalError(fmt.Errorf("failed to allocate a serviceIP: %v", err)) + } + allocated[family] = allocatedIP.String() + } else { + parsedIP := net.ParseIP(ip) + if err := allocator.Allocate(parsedIP); err != nil { + el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs"), service.Spec.ClusterIPs, fmt.Sprintf("failed to allocated ip:%v with error:%v", ip, err))} + return allocated, errors.NewInvalid(api.Kind("Service"), service.Name, el) + } + allocated[family] = ip + } } - - return r.serviceIPs + return allocated, nil } -func (r *REST) getAllocatorBySpec(service *api.Service) ipallocator.Interface { - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || - service.Spec.IPFamily == nil || - r.secondaryServiceIPs == nil { - return r.serviceIPs +// releases clusterIPs per family +func (rs *REST) releaseClusterIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) { + if toRelease == nil { + return nil, nil } - secondaryAllocatorCIDR := r.secondaryServiceIPs.CIDR() - if (*(service.Spec.IPFamily) == api.IPv6Protocol) == netutil.IsIPv6CIDR(&secondaryAllocatorCIDR) { - return r.secondaryServiceIPs + released := make(map[api.IPFamily]string) + for family, ip := range toRelease { + allocator, ok := rs.serviceIPAllocatorsByFamily[family] + if !ok { + // cluster was configured for dual stack, then single stack + klog.V(4).Infof("delete service. Not releasing ClusterIP:%v because IPFamily:%v is no longer configured on server", ip, family) + continue + } + + parsedIP := net.ParseIP(ip) + if err := allocator.Release(parsedIP); err != nil { + return released, err + } + released[family] = ip } - return r.serviceIPs + return released, nil +} + +// standard allocator for dualstackgate==Off, hard wired dependency +// and ignores policy, families and clusterIPs +func (rs *REST) allocServiceClusterIP(service *api.Service) (map[api.IPFamily]string, error) { + toAlloc := make(map[api.IPFamily]string) + + // get clusterIP.. empty string if user did not specify an ip + toAlloc[rs.defaultServiceIPFamily] = service.Spec.ClusterIP + // alloc + allocated, err := rs.allocClusterIPs(service, toAlloc) + + // set + if err == nil { + service.Spec.ClusterIP = allocated[rs.defaultServiceIPFamily] + service.Spec.ClusterIPs = []string{allocated[rs.defaultServiceIPFamily]} + } + + return allocated, err +} + +// allocates ClusterIPs for a service +func (rs *REST) allocServiceClusterIPs(service *api.Service) (map[api.IPFamily]string, error) { + // external name don't get ClusterIPs + if service.Spec.Type == api.ServiceTypeExternalName { + return nil, nil + } + + // headless don't get ClusterIPs + if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone { + return nil, nil + } + + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + return rs.allocServiceClusterIP(service) + } + + toAlloc := make(map[api.IPFamily]string) + // at this stage, the only fact we know is that service has correct ip families + // assigned to it. It may have partial assigned ClusterIPs (Upgrade to dual stack) + // may have no ips at all. The below loop is meant to fix this + // (we also know that this cluster has these families) + + // if there is no slice to work with + if service.Spec.ClusterIPs == nil { + service.Spec.ClusterIPs = make([]string, 0, len(service.Spec.IPFamilies)) + } + + for i, ipFamily := range service.Spec.IPFamilies { + if i > (len(service.Spec.ClusterIPs) - 1) { + service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* just a marker */) + } + + toAlloc[ipFamily] = service.Spec.ClusterIPs[i] + } + + // allocate + allocated, err := rs.allocClusterIPs(service, toAlloc) + + // set if successful + if err == nil { + for family, ip := range allocated { + for i, check := range service.Spec.IPFamilies { + if family == check { + service.Spec.ClusterIPs[i] = ip + // while we technically don't need to do that testing rest does not + // go through conversion logic but goes through validation *sigh*. + // so we set ClusterIP here as well + // because the testing code expects valid (as they are output-ed from conversion) + // as it patches fields + if i == 0 { + service.Spec.ClusterIP = ip + } + } + } + } + } + + return allocated, err +} + +// handles type change/upgrade/downgrade change type for an update service +// this func does not perform actual release of clusterIPs. it returns +// a map[family]ip for the caller to release when everything else has +// executed successfully +func (rs *REST) handleClusterIPsForUpdatedService(oldService *api.Service, service *api.Service) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) { + // use cases: + // A: service changing types from ExternalName TO ClusterIP types ==> allocate all new + // B: service changing types from ClusterIP types TO ExternalName ==> release all allocated + // C: Service upgrading to dual stack ==> partial allocation + // D: service downgrading from dual stack ==> partial release + + // CASE A: + // Update service from ExternalName to non-ExternalName, should initialize ClusterIP. + if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName { + allocated, err := rs.allocServiceClusterIPs(service) + return allocated, nil, err + } + + // CASE B: + // Update service from non-ExternalName to ExternalName, should release ClusterIP if exists. + if oldService.Spec.Type != api.ServiceTypeExternalName && service.Spec.Type == api.ServiceTypeExternalName { + toRelease = make(map[api.IPFamily]string) + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + // for non dual stack enabled cluster we use clusterIPs + toRelease[rs.defaultServiceIPFamily] = oldService.Spec.ClusterIP + } else { + // dual stack is enabled, collect ClusterIPs by families + for i, family := range oldService.Spec.IPFamilies { + toRelease[family] = oldService.Spec.ClusterIPs[i] + } + } + + return nil, toRelease, nil + } + + // if headless service then we bail out early (no clusterIPs management needed) + if len(oldService.Spec.ClusterIPs) > 0 && oldService.Spec.ClusterIPs[0] == api.ClusterIPNone { + return nil, nil, nil + } + + // upgrade and downgrade are specific to dualstack + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + return nil, nil, nil + } + + upgraded := len(oldService.Spec.IPFamilies) == 1 && len(service.Spec.IPFamilies) == 2 + downgraded := len(oldService.Spec.IPFamilies) == 2 && len(service.Spec.IPFamilies) == 1 + + // CASE C: + if upgraded { + toAllocate := make(map[api.IPFamily]string) + // if secondary ip was named, just get it. if not add a marker + if len(service.Spec.ClusterIPs) < 2 { + service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* marker */) + } + + toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1] + + // allocate + allocated, err := rs.allocClusterIPs(service, toAllocate) + // set if successful + if err == nil { + service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]] + } + + return allocated, nil, err + } + + // CASE D: + if downgraded { + toRelease = make(map[api.IPFamily]string) + toRelease[oldService.Spec.IPFamilies[1]] = oldService.Spec.ClusterIPs[1] + // note: we don't release clusterIP, this is left to clean up in the action itself + return nil, toRelease, err + } + // it was not an upgrade nor downgrade + return nil, nil, nil +} + +// for pre dual stack (gate == off). Hardwired to ClusterIP and ignores all new fields +func (rs *REST) releaseServiceClusterIP(service *api.Service) (released map[api.IPFamily]string, err error) { + toRelease := make(map[api.IPFamily]string) + + // we need to do that to handle cases where allocator is no longer configured on + // cluster + if netutil.IsIPv6String(service.Spec.ClusterIP) { + toRelease[api.IPv6Protocol] = service.Spec.ClusterIP + } else { + toRelease[api.IPv4Protocol] = service.Spec.ClusterIP + } + + return rs.releaseClusterIPs(toRelease) +} + +// releases allocated ClusterIPs for service that is about to be deleted +func (rs *REST) releaseServiceClusterIPs(service *api.Service) (released map[api.IPFamily]string, err error) { + // external name don't get ClusterIPs + if service.Spec.Type == api.ServiceTypeExternalName { + return nil, nil + } + + // headless don't get ClusterIPs + if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone { + return nil, nil + } + + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + return rs.releaseServiceClusterIP(service) + } + + toRelease := make(map[api.IPFamily]string) + for _, ip := range service.Spec.ClusterIPs { + if netutil.IsIPv6String(ip) { + toRelease[api.IPv6Protocol] = ip + } else { + toRelease[api.IPv4Protocol] = ip + } + } + return rs.releaseClusterIPs(toRelease) +} + +// attempts to default service ip families according to cluster configuration +// while ensuring that provided families are configured on cluster. +func (rs *REST) tryDefaultValidateServiceClusterIPFields(service *api.Service) error { + // can not do anything here + if service.Spec.Type == api.ServiceTypeExternalName { + return nil + } + + // gate off. We don't need to validate or default new fields + // we totally depend on existing validation in apis/validation + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + return nil + } + + // two families or two IPs with SingleStack + if service.Spec.IPFamilyPolicy != nil { + el := make(field.ErrorList, 0) + if *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicySingleStack { + if len(service.Spec.ClusterIPs) == 2 { + el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "must be RequireDualStack or PreferDualStack when multiple 'clusterIPs' are specified")) + } + if len(service.Spec.IPFamilies) == 2 { + el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "must be RequireDualStack or PreferDualStack when multiple 'ipFamilies' are specified")) + } + } + + if len(el) > 0 { + return errors.NewInvalid(api.Kind("Service"), service.Name, el) + } + } + + // default families according to cluster IPs + for i, ip := range service.Spec.ClusterIPs { + if ip == api.ClusterIPNone { + break + } + + // we have previously validated for ip correctness and if family exist it will match ip family + // so the following is safe to do + isIPv6 := netutil.IsIPv6String(ip) + + // family is not there. + if i > len(service.Spec.IPFamilies)-1 { + if isIPv6 { + // first make sure that family(ip) is configured + if _, found := rs.serviceIPAllocatorsByFamily[api.IPv6Protocol]; !found { + el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv6 on a cluster which is not configured for it")} + return errors.NewInvalid(api.Kind("Service"), service.Name, el) + } + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol) + } else { + // first make sure that family(ip) is configured + if _, found := rs.serviceIPAllocatorsByFamily[api.IPv4Protocol]; !found { + el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not use IPv4 on a cluster which is not configured for it")} + return errors.NewInvalid(api.Kind("Service"), service.Name, el) + } + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol) + } + } + } + + // default headless+selectorless + if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone && len(service.Spec.Selector) == 0 { + + if service.Spec.IPFamilyPolicy == nil { + requireDualStack := api.IPFamilyPolicyRequireDualStack + service.Spec.IPFamilyPolicy = &requireDualStack + } + + // if not set by user + if len(service.Spec.IPFamilies) == 0 { + service.Spec.IPFamilies = []api.IPFamily{rs.defaultServiceIPFamily} + } + + // this follows headful services. With one exception on a single stack + // cluster the user is allowed to create headless services that has multi families + // the validation allows it + if len(service.Spec.IPFamilies) < 2 { + if *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack || + (*(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyPreferDualStack && len(rs.serviceIPAllocatorsByFamily) == 2) { + // add the alt ipfamily + if service.Spec.IPFamilies[0] == api.IPv4Protocol { + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol) + } else { + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol) + } + } + } + + // nothing more needed here + return nil + } + + // ipfamily check + // the following applies on all type of services including headless w/ selector + el := make(field.ErrorList, 0) + + // asking for dual stack on a non dual stack cluster + // should fail without assigning any family + if service.Spec.IPFamilyPolicy != nil && *(service.Spec.IPFamilyPolicy) == api.IPFamilyPolicyRequireDualStack && len(rs.serviceIPAllocatorsByFamily) < 2 { + el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy, "Cluster is not configured for dual stack services")) + } + + // if there is a family requested then it has to be configured on cluster + for i, ipFamily := range service.Spec.IPFamilies { + if _, found := rs.serviceIPAllocatorsByFamily[ipFamily]; !found { + el = append(el, field.Invalid(field.NewPath("spec", "ipFamilies").Index(i), service.Spec.ClusterIPs, fmt.Sprintf("ipfamily %v is not configured on cluster", ipFamily))) + } + } + + // if we have validation errors return them and bail out + if len(el) > 0 { + return errors.NewInvalid(api.Kind("Service"), service.Name, el) + } + + // default ipFamilyPolicy to SingleStack. if there are + // web hooks, they must have already ran by now + if service.Spec.IPFamilyPolicy == nil { + singleStack := api.IPFamilyPolicySingleStack + service.Spec.IPFamilyPolicy = &singleStack + } + + // nil families, gets cluster default (if feature flag is not in effect, the strategy will take care of removing it) + if len(service.Spec.IPFamilies) == 0 { + service.Spec.IPFamilies = []api.IPFamily{rs.defaultServiceIPFamily} + } + + // is this service looking for dual stack, and this cluster does have two families? + // if so, then append the missing family + if *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && + len(service.Spec.IPFamilies) == 1 && + len(rs.serviceIPAllocatorsByFamily) == 2 { + + if service.Spec.IPFamilies[0] == api.IPv4Protocol { + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol) + } + + if service.Spec.IPFamilies[0] == api.IPv6Protocol { + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol) + } + } + + return nil } func isValidAddress(ctx context.Context, addr *api.EndpointAddress, pods rest.Getter) error { @@ -653,48 +1063,6 @@ func allocateHealthCheckNodePort(service *api.Service, nodePortOp *portallocator return nil } -// The return bool value indicates if a cluster IP is allocated successfully. -func initClusterIP(service *api.Service, allocator ipallocator.Interface) (bool, error) { - var allocatedIP net.IP - - switch { - case service.Spec.ClusterIP == "": - // Allocate next available. - ip, err := allocator.AllocateNext() - if err != nil { - // TODO: what error should be returned here? It's not a - // field-level validation failure (the field is valid), and it's - // not really an internal error. - return false, errors.NewInternalError(fmt.Errorf("failed to allocate a serviceIP: %v", err)) - } - allocatedIP = ip - service.Spec.ClusterIP = ip.String() - case service.Spec.ClusterIP != api.ClusterIPNone && service.Spec.ClusterIP != "": - // Try to respect the requested IP. - ip := net.ParseIP(service.Spec.ClusterIP) - if err := allocator.Allocate(ip); err != nil { - // TODO: when validation becomes versioned, this gets more complicated. - el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIP"), service.Spec.ClusterIP, err.Error())} - return false, errors.NewInvalid(api.Kind("Service"), service.Name, el) - } - allocatedIP = ip - } - - // assuming the object was valid prior to setting, always force the IPFamily - // to match the allocated IP at this point - if allocatedIP != nil { - if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - ipFamily := api.IPv4Protocol - if netutil.IsIPv6(allocatedIP) { - ipFamily = api.IPv6Protocol - } - service.Spec.IPFamily = &ipFamily - } - } - - return allocatedIP != nil, nil -} - func initNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error { svcPortToNodePort := map[int]int{} for i := range service.Spec.Ports { diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go index c5cf97a62af..a33f3d43d14 100644 --- a/pkg/registry/core/service/storage/rest_test.go +++ b/pkg/registry/core/service/storage/rest_test.go @@ -28,7 +28,6 @@ import ( metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/intstr" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/watch" @@ -49,13 +48,12 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" + + netutil "k8s.io/utils/net" ) var ( - singleStackIPv4 = []api.IPFamily{api.IPv4Protocol} - singleStackIPv6 = []api.IPFamily{api.IPv6Protocol} - dualStackIPv4Primary = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol} - dualStackIPv6Primary = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol} + singleStackIPv4 = []api.IPFamily{api.IPv4Protocol} ) // TODO(wojtek-t): Cleanup this file. @@ -271,13 +269,6 @@ func makeIPNet6(t *testing.T) *net.IPNet { return net } -func ipnetGet(t *testing.T, useIPv6 bool) *net.IPNet { - if useIPv6 { - return makeIPNet6(t) - } - return makeIPNet(t) -} - func releaseServiceNodePorts(t *testing.T, ctx context.Context, svcName string, rest *REST, registry ServiceStorage) { obj, err := registry.Get(ctx, svcName, &metav1.GetOptions{}) if err != nil { @@ -298,20 +289,17 @@ func releaseServiceNodePorts(t *testing.T, ctx context.Context, svcName string, } func TestServiceRegistryCreate(t *testing.T) { - ipv4Service := api.IPv4Protocol - ipv6Service := api.IPv6Protocol - testCases := []struct { svc *api.Service name string families []api.IPFamily enableDualStack bool - expectErr string }{ { name: "Service IPFamily default cluster dualstack:off", enableDualStack: false, - families: singleStackIPv4, + families: []api.IPFamily{api.IPv4Protocol}, + svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ @@ -327,15 +315,16 @@ func TestServiceRegistryCreate(t *testing.T) { }, }, { - name: "Service IPFamily:v4 dualstack off", - families: singleStackIPv4, + name: "Service IPFamily:v4 dualstack off", + enableDualStack: false, + families: []api.IPFamily{api.IPv4Protocol}, svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -347,14 +336,14 @@ func TestServiceRegistryCreate(t *testing.T) { { name: "Service IPFamily:v4 dualstack on", enableDualStack: true, - families: singleStackIPv4, + families: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -366,14 +355,15 @@ func TestServiceRegistryCreate(t *testing.T) { { name: "Service IPFamily:v6 dualstack on", enableDualStack: true, - families: singleStackIPv6, + families: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -382,64 +372,16 @@ func TestServiceRegistryCreate(t *testing.T) { }, }, }, - { - name: "Service IPFamily:4 dualstack on, single stack ipv6, service CIDR IPv6", - enableDualStack: true, - families: singleStackIPv6, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - expectErr: "Service \"foo\" is invalid: spec.ipFamily: Invalid value: \"IPv4\": only the following families are allowed: IPv6", - }, - { - name: "Service IP:4 dualstack on, single stack ipv6, service CIDR IPv6", - enableDualStack: true, - families: singleStackIPv6, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - ClusterIP: "10.0.30.0", - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - expectErr: "Service \"foo\" is invalid: spec.ipFamily: Invalid value: \"IPv6\": does not match IPv4 cluster IP", - }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - storage, registry, server := NewTestREST(t, nil, tc.families) defer server.Terminate(t) ctx := genericapirequest.NewDefaultContext() createdSvc, err := storage.Create(ctx, tc.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) - if (len(tc.expectErr) > 0) != (err != nil) { - t.Fatalf("unexpected error: %v", err) - } if err != nil { - if !strings.Contains(err.Error(), tc.expectErr) { - t.Fatalf("unexpected error message: %v", err) - } - return + t.Fatalf("error creating service %v", err) } createdService := createdSvc.(*api.Service) objMeta, err := meta.Accessor(createdService) @@ -455,10 +397,14 @@ func TestServiceRegistryCreate(t *testing.T) { if createdService.CreationTimestamp.IsZero() { t.Errorf("Expected timestamp to be set, got: %v", createdService.CreationTimestamp) } - allocNet := ipnetGet(t, tc.families[0] == api.IPv6Protocol) - if !allocNet.Contains(net.ParseIP(createdService.Spec.ClusterIP)) { - t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIP) + for i, family := range createdService.Spec.IPFamilies { + allocator := storage.serviceIPAllocatorsByFamily[family] + c := allocator.CIDR() + cidr := &c + if !cidr.Contains(net.ParseIP(createdService.Spec.ClusterIPs[i])) { + t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIPs[i]) + } } srv, err := registry.GetService(ctx, tc.svc.Name, &metav1.GetOptions{}) if err != nil { @@ -472,17 +418,15 @@ func TestServiceRegistryCreate(t *testing.T) { } func TestServiceRegistryCreateDryRun(t *testing.T) { - ipv6Service := api.IPv6Protocol + requireDualStack := api.IPFamilyPolicyRequireDualStack testCases := []struct { name string svc *api.Service enableDualStack bool - families []api.IPFamily }{ { name: "v4 service featuregate off", enableDualStack: false, - families: singleStackIPv4, svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ @@ -490,45 +434,7 @@ func TestServiceRegistryCreateDryRun(t *testing.T) { SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, ClusterIP: "1.2.3.4", - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - }, - { - name: "v4 service featuregate on but singlestack", - enableDualStack: true, - families: singleStackIPv4, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - ClusterIP: "1.2.3.4", - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - }, - { - name: "v6 service featuregate off", - enableDualStack: false, - families: singleStackIPv6, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - ClusterIP: "2000:0:0:0:0:0:0:1", + ClusterIPs: []string{"1.2.3.4"}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -540,15 +446,15 @@ func TestServiceRegistryCreateDryRun(t *testing.T) { { name: "v6 service featuregate on but singlestack", enableDualStack: true, - families: singleStackIPv6, svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, ClusterIP: "2000:0:0:0:0:0:0:1", + ClusterIPs: []string{"2000:0:0:0:0:0:0:1"}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -557,18 +463,19 @@ func TestServiceRegistryCreateDryRun(t *testing.T) { }, }, }, - { - name: "v4 service dualstack ipv4 primary", + name: "dualstack v4,v6 service", enableDualStack: true, - families: dualStackIPv4Primary, svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4", "2000:0:0:0:0:0:0:1"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -577,60 +484,19 @@ func TestServiceRegistryCreateDryRun(t *testing.T) { }, }, }, - { - name: "v4 service dualstack ipv6 primary", + name: "dualstack v6,v4 service", enableDualStack: true, - families: dualStackIPv6Primary, svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &singleStackIPv4[0], - ClusterIP: "1.2.3.4", - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - }, - - { - name: "v6 service dualstack ipv4 primary", - enableDualStack: true, - families: dualStackIPv4Primary, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - ClusterIP: "2000:0:0:0:0:0:0:1", - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - }, - { - name: "v6 service dualstack ipv6 primary", - enableDualStack: true, - families: dualStackIPv6Primary, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, + IPFamilyPolicy: &requireDualStack, ClusterIP: "2000:0:0:0:0:0:0:1", + ClusterIPs: []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -644,7 +510,12 @@ func TestServiceRegistryCreateDryRun(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - storage, registry, server := NewTestREST(t, nil, tc.families) + + families := []api.IPFamily{api.IPv4Protocol} + if tc.enableDualStack { + families = append(families, api.IPv6Protocol) + } + storage, registry, server := NewTestREST(t, nil, families) defer server.Terminate(t) ctx := genericapirequest.NewDefaultContext() @@ -653,12 +524,10 @@ func TestServiceRegistryCreateDryRun(t *testing.T) { t.Fatalf("Unexpected error: %v", err) } - if storage.serviceIPs.Has(net.ParseIP(tc.svc.Spec.ClusterIP)) { - t.Errorf("unexpected side effect: ip allocated") - } - if storage.secondaryServiceIPs != nil { - if storage.secondaryServiceIPs.Has(net.ParseIP(tc.svc.Spec.ClusterIP)) { - t.Errorf("unexpected side effect: secondary ip allocated") + for i, family := range tc.svc.Spec.IPFamilies { + alloc := storage.serviceIPAllocatorsByFamily[family] + if alloc.Has(net.ParseIP(tc.svc.Spec.ClusterIPs[i])) { + t.Errorf("unexpected side effect: ip allocated %v", tc.svc.Spec.ClusterIPs[i]) } } @@ -977,184 +846,52 @@ func TestServiceStorageValidatesCreate(t *testing.T) { } func TestServiceRegistryUpdate(t *testing.T) { - testCases := []struct { - name string - in *api.Service - update func(*api.Service) - out *api.Service + ctx := genericapirequest.NewDefaultContext() + storage, registry, server := NewTestREST(t, nil, []api.IPFamily{api.IPv4Protocol}) + defer server.Terminate(t) - enableDualStack bool - families []api.IPFamily - - expect *api.Service - expectErr string - }{ - { - name: "simple update", - families: singleStackIPv4, // not actually relevant to test - in: &api.Service{ - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz1"}, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - update: func(svc *api.Service) { - svc.Spec = api.ServiceSpec{ - Selector: map[string]string{"bar": "baz2"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - } - }, - }, - { - name: "ipv4: update to service that drops IPFamily succeeds", - enableDualStack: true, - families: singleStackIPv4, - in: &api.Service{ - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz1"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - update: func(svc *api.Service) { - svc.Spec.IPFamily = nil - }, - out: &api.Service{ - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz1"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &singleStackIPv4[0], - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - }, - { - name: "ipv6: update to service that drops IPFamily succeeds", - enableDualStack: true, - families: singleStackIPv6, - in: &api.Service{ - Spec: api.ServiceSpec{ - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &singleStackIPv6[0], - Selector: map[string]string{"bar": "baz1"}, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - update: func(svc *api.Service) { - svc.Spec.IPFamily = nil - }, - out: &api.Service{ - Spec: api.ServiceSpec{ - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &singleStackIPv6[0], - Selector: map[string]string{"bar": "baz1"}, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - }, - { - name: "ipv6: changing IPFamily fails", - enableDualStack: true, - families: singleStackIPv6, - in: &api.Service{ - Spec: api.ServiceSpec{ - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &singleStackIPv6[0], - Selector: map[string]string{"bar": "baz1"}, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - update: func(svc *api.Service) { - svc.Spec.IPFamily = &singleStackIPv4[0] - }, - expectErr: "spec.ipFamily: Invalid value: \"IPv4\": field is immutable, spec.ipFamily: Invalid value: \"IPv4\": only the following families are allowed: IPv6", + _, err := registry.Create(ctx, &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + SessionAffinity: api.ServiceAffinityNone, + Selector: map[string]string{"bar": "baz1"}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, }, + }, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Expected no error: %v", err) } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - ctx := genericapirequest.NewDefaultContext() - storage, registry, server := NewTestREST(t, nil, tc.families) - defer server.Terminate(t) - tc.in.ObjectMeta = metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault} + obj, err := storage.Get(ctx, "foo", &metav1.GetOptions{}) + if err != nil { + t.Fatalf("unexpected error :%v", err) + } + svc := obj.(*api.Service) - obj, err := registry.Create(ctx, tc.in, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) - svc := obj.(*api.Service) - if err != nil { - t.Fatalf("Expected no error: %v", err) - } - t.Logf("%#v", svc) - tc.update(svc) - updatedSvc, created, err := storage.Update(ctx, "foo", rest.DefaultUpdatedObjectInfo(svc), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) - if (len(tc.expectErr) > 0) != (err != nil) { - t.Fatalf("unxpected error: %v", err) - } - if err != nil { - if !strings.Contains(err.Error(), tc.expectErr) { - t.Fatalf("unexpected error: %v", err) - } - return - } - if updatedSvc == nil { - t.Errorf("Expected non-nil object") - } - if created { - t.Errorf("expected not created") - } - updatedService := updatedSvc.(*api.Service) - if updatedService.Name != "foo" { - t.Errorf("Expected foo, but got %v", updatedService.Name) - } - - expected := svc - if tc.out != nil { - expected = tc.out - } - expected.ObjectMeta = updatedService.ObjectMeta - if !reflect.DeepEqual(expected, updatedSvc) { - t.Fatalf("unexpected object: %s", diff.ObjectReflectDiff(expected, updatedService)) - } - if e, a := "foo", registry.UpdatedID; e != a { - t.Errorf("Expected %v, but got %v", e, a) - } - }) + // update selector + svc.Spec.Selector = map[string]string{"bar": "baz2"} + updatedSvc, created, err := storage.Update(ctx, "foo", rest.DefaultUpdatedObjectInfo(svc), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Expected no error: %v", err) + } + if updatedSvc == nil { + t.Errorf("Expected non-nil object") + } + if created { + t.Errorf("expected not created") + } + updatedService := updatedSvc.(*api.Service) + if updatedService.Name != "foo" { + t.Errorf("Expected foo, but got %v", updatedService.Name) + } + if e, a := "foo", registry.UpdatedID; e != a { + t.Errorf("Expected %v, but got %v", e, a) } } @@ -1225,6 +962,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) { SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -1235,7 +973,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) { if err != nil { t.Fatalf("Expected no error: %v", err) } - if storage.serviceIPs.Has(net.ParseIP("1.2.3.4")) { + if storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily].Has(net.ParseIP("1.2.3.4")) { t.Errorf("unexpected side effect: ip allocated") } @@ -1247,6 +985,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) { SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeNodePort, ClusterIP: "1.2.3.5", + ClusterIPs: []string{"1.2.3.5"}, Ports: []api.ServicePort{{ NodePort: 30020, Port: 6502, @@ -1291,6 +1030,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) { SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -1321,7 +1061,7 @@ func TestServiceRegistryUpdateDryRun(t *testing.T) { if err != nil { t.Fatalf("Expected no error: %v", err) } - if !storage.serviceIPs.Has(net.ParseIP("1.2.3.4")) { + if !storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily].Has(net.ParseIP("1.2.3.4")) { t.Errorf("unexpected side effect: ip unallocated") } } @@ -1454,6 +1194,7 @@ func TestServiceRegistryDeleteDryRun(t *testing.T) { SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -1472,7 +1213,7 @@ func TestServiceRegistryDeleteDryRun(t *testing.T) { if e, a := "", registry.DeletedID; e != a { t.Errorf("Expected %v, but got %v", e, a) } - if !storage.serviceIPs.Has(net.ParseIP("1.2.3.4")) { + if !storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily].Has(net.ParseIP("1.2.3.4")) { t.Errorf("unexpected side effect: ip unallocated") } @@ -1495,6 +1236,9 @@ func TestServiceRegistryDeleteDryRun(t *testing.T) { if err != nil { t.Fatalf("Expected no error: %v", err) } + + isValidClusterIPFields(t, storage, svc, svc) + _, _, err = storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}}) if err != nil { t.Fatalf("Expected no error: %v", err) @@ -1505,6 +1249,48 @@ func TestServiceRegistryDeleteDryRun(t *testing.T) { if !storage.serviceNodePorts.Has(30030) { t.Errorf("unexpected side effect: NodePort unallocated") } + + // dry run for non dualstack + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + dualstack_storage, dualstack_registry, dualstack_server := NewTestREST(t, nil, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}) + defer dualstack_server.Terminate(t) + requireDualStack := api.IPFamilyPolicyRequireDualStack + // Test dry run delete request with cluster ip + dualstack_svc := &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIP: "2000:0:0:0:0:0:0:1", + ClusterIPs: []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + } + + _, err = dualstack_storage.Create(ctx, dualstack_svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Expected no error: %v", err) + } + isValidClusterIPFields(t, dualstack_storage, dualstack_svc, dualstack_svc) + _, _, err = dualstack_storage.Delete(ctx, dualstack_svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}}) + if err != nil { + t.Fatalf("Expected no error: %v", err) + } + if e, a := "", dualstack_registry.DeletedID; e != a { + t.Errorf("Expected %v, but got %v", e, a) + } + for i, family := range dualstack_svc.Spec.IPFamilies { + if !dualstack_storage.serviceIPAllocatorsByFamily[family].Has(net.ParseIP(dualstack_svc.Spec.ClusterIPs[i])) { + t.Errorf("unexpected side effect: ip unallocated %v", dualstack_svc.Spec.ClusterIPs[i]) + } + } } func TestServiceRegistryDeleteExternal(t *testing.T) { @@ -1915,8 +1701,8 @@ func TestServiceRegistryIPAllocation(t *testing.T) { if createdService1.Name != "foo" { t.Errorf("Expected foo, but got %v", createdService1.Name) } - if !makeIPNet(t).Contains(net.ParseIP(createdService1.Spec.ClusterIP)) { - t.Errorf("Unexpected ClusterIP: %s", createdService1.Spec.ClusterIP) + if !makeIPNet(t).Contains(net.ParseIP(createdService1.Spec.ClusterIPs[0])) { + t.Errorf("Unexpected ClusterIP: %s", createdService1.Spec.ClusterIPs[0]) } svc2 := &api.Service{ @@ -1937,14 +1723,14 @@ func TestServiceRegistryIPAllocation(t *testing.T) { if createdService2.Name != "bar" { t.Errorf("Expected bar, but got %v", createdService2.Name) } - if !makeIPNet(t).Contains(net.ParseIP(createdService2.Spec.ClusterIP)) { - t.Errorf("Unexpected ClusterIP: %s", createdService2.Spec.ClusterIP) + if !makeIPNet(t).Contains(net.ParseIP(createdService2.Spec.ClusterIPs[0])) { + t.Errorf("Unexpected ClusterIP: %s", createdService2.Spec.ClusterIPs[0]) } testIPs := []string{"1.2.3.93", "1.2.3.94", "1.2.3.95", "1.2.3.96"} testIP := "" for _, ip := range testIPs { - if !storage.serviceIPs.(*ipallocator.Range).Has(net.ParseIP(ip)) { + if !storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily].(*ipallocator.Range).Has(net.ParseIP(ip)) { testIP = ip break } @@ -1955,6 +1741,7 @@ func TestServiceRegistryIPAllocation(t *testing.T) { Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, ClusterIP: testIP, + ClusterIPs: []string{testIP}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, Ports: []api.ServicePort{{ @@ -1970,8 +1757,8 @@ func TestServiceRegistryIPAllocation(t *testing.T) { t.Fatal(err) } createdService3 := createdSvc3.(*api.Service) - if createdService3.Spec.ClusterIP != testIP { // specific IP - t.Errorf("Unexpected ClusterIP: %s", createdService3.Spec.ClusterIP) + if createdService3.Spec.ClusterIPs[0] != testIP { // specific IP + t.Errorf("Unexpected ClusterIP: %s", createdService3.Spec.ClusterIPs[0]) } } @@ -1998,8 +1785,8 @@ func TestServiceRegistryIPReallocation(t *testing.T) { if createdService1.Name != "foo" { t.Errorf("Expected foo, but got %v", createdService1.Name) } - if !makeIPNet(t).Contains(net.ParseIP(createdService1.Spec.ClusterIP)) { - t.Errorf("Unexpected ClusterIP: %s", createdService1.Spec.ClusterIP) + if !makeIPNet(t).Contains(net.ParseIP(createdService1.Spec.ClusterIPs[0])) { + t.Errorf("Unexpected ClusterIP: %s", createdService1.Spec.ClusterIPs[0]) } _, _, err := storage.Delete(ctx, createdService1.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) @@ -2026,8 +1813,8 @@ func TestServiceRegistryIPReallocation(t *testing.T) { if createdService2.Name != "bar" { t.Errorf("Expected bar, but got %v", createdService2.Name) } - if !makeIPNet(t).Contains(net.ParseIP(createdService2.Spec.ClusterIP)) { - t.Errorf("Unexpected ClusterIP: %s", createdService2.Spec.ClusterIP) + if !makeIPNet(t).Contains(net.ParseIP(createdService2.Spec.ClusterIPs[0])) { + t.Errorf("Unexpected ClusterIP: %s", createdService2.Spec.ClusterIPs[0]) } } @@ -2054,14 +1841,17 @@ func TestServiceRegistryIPUpdate(t *testing.T) { if createdService.Spec.Ports[0].Port != 6502 { t.Errorf("Expected port 6502, but got %v", createdService.Spec.Ports[0].Port) } - if !makeIPNet(t).Contains(net.ParseIP(createdService.Spec.ClusterIP)) { - t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIP) + if !makeIPNet(t).Contains(net.ParseIP(createdService.Spec.ClusterIPs[0])) { + t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIPs[0]) } update := createdService.DeepCopy() update.Spec.Ports[0].Port = 6503 - updatedSvc, _, _ := storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + updatedSvc, _, errUpdate := storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + if errUpdate != nil { + t.Fatalf("unexpected error during update %v", errUpdate) + } updatedService := updatedSvc.(*api.Service) if updatedService.Spec.Ports[0].Port != 6503 { t.Errorf("Expected port 6503, but got %v", updatedService.Spec.Ports[0].Port) @@ -2070,7 +1860,7 @@ func TestServiceRegistryIPUpdate(t *testing.T) { testIPs := []string{"1.2.3.93", "1.2.3.94", "1.2.3.95", "1.2.3.96"} testIP := "" for _, ip := range testIPs { - if !storage.serviceIPs.(*ipallocator.Range).Has(net.ParseIP(ip)) { + if !storage.serviceIPAllocatorsByFamily[storage.defaultServiceIPFamily].(*ipallocator.Range).Has(net.ParseIP(ip)) { testIP = ip break } @@ -2078,7 +1868,8 @@ func TestServiceRegistryIPUpdate(t *testing.T) { update = createdService.DeepCopy() update.Spec.Ports[0].Port = 6503 - update.Spec.ClusterIP = testIP // Error: Cluster IP is immutable + update.Spec.ClusterIP = testIP + update.Spec.ClusterIPs[0] = testIP _, _, err := storage.Update(ctx, update.Name, rest.DefaultUpdatedObjectInfo(update), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) if err == nil || !errors.IsInvalid(err) { @@ -2114,8 +1905,8 @@ func TestServiceRegistryIPLoadBalancer(t *testing.T) { if createdService.Spec.Ports[0].Port != 6502 { t.Errorf("Expected port 6502, but got %v", createdService.Spec.Ports[0].Port) } - if !makeIPNet(t).Contains(net.ParseIP(createdService.Spec.ClusterIP)) { - t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIP) + if !makeIPNet(t).Contains(net.ParseIP(createdService.Spec.ClusterIPs[0])) { + t.Errorf("Unexpected ClusterIP: %s", createdService.Spec.ClusterIPs[0]) } update := createdService.DeepCopy() @@ -2297,22 +2088,22 @@ func TestServiceRegistryExternalTrafficGlobal(t *testing.T) { } func TestInitClusterIP(t *testing.T) { - ipv4Service := api.IPv4Protocol - ipv6Service := api.IPv6Protocol + singleStack := api.IPFamilyPolicySingleStack + requireDualStack := api.IPFamilyPolicyRequireDualStack + preferDualStack := api.IPFamilyPolicyPreferDualStack + testCases := []struct { name string svc *api.Service - enableDualStack bool - families []api.IPFamily - useSecondaryAlloc bool - - expectClusterIP bool - expectedAllocatedIP string + enableDualStackAllocator bool + preAllocateClusterIPs map[api.IPFamily]string + expectError bool + expectedCountIPs int + expectedClusterIPs []string }{ { - name: "Allocate new ClusterIP", - families: singleStackIPv4, + name: "Allocate single stack ClusterIP (v4)", svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ @@ -2326,20 +2117,20 @@ func TestInitClusterIP(t *testing.T) { }}, }, }, - expectClusterIP: true, + enableDualStackAllocator: false, + expectError: false, + preAllocateClusterIPs: nil, + expectedCountIPs: 1, }, { - name: "Allocate new ClusterIP-v6 dualstack ipv4 primary", - useSecondaryAlloc: true, - enableDualStack: true, - families: dualStackIPv4Primary, + name: "Allocate single ClusterIP (v6)", svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -2347,61 +2138,22 @@ func TestInitClusterIP(t *testing.T) { }}, }, }, - expectClusterIP: true, + expectError: false, + enableDualStackAllocator: true, + preAllocateClusterIPs: nil, + expectedCountIPs: 1, }, { - name: "Allocate new ClusterIP-v6 dualstack ipv6 primary", - useSecondaryAlloc: true, - enableDualStack: true, - families: dualStackIPv6Primary, + name: "Allocate specified ClusterIP (v4)", svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - expectClusterIP: true, - }, - { - name: "Allocate new ClusterIP-v6 single stack", - families: singleStackIPv6, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - Ports: []api.ServicePort{{ - Port: 6502, - Protocol: api.ProtocolTCP, - TargetPort: intstr.FromInt(6502), - }}, - }, - }, - expectClusterIP: true, - }, - { - name: "Allocate specified ClusterIP", - enableDualStack: true, - useSecondaryAlloc: true, - families: dualStackIPv6Primary, - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -2409,21 +2161,23 @@ func TestInitClusterIP(t *testing.T) { }}, }, }, - expectClusterIP: true, - expectedAllocatedIP: "1.2.3.4", + expectError: false, + enableDualStackAllocator: true, + preAllocateClusterIPs: nil, + expectedCountIPs: 1, + expectedClusterIPs: []string{"1.2.3.4"}, }, { - name: "Allocate specified ClusterIP-v6", - enableDualStack: true, - families: dualStackIPv6Primary, + name: "Allocate specified ClusterIP-v6", svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, ClusterIP: "2000:0:0:0:0:0:0:1", + ClusterIPs: []string{"2000:0:0:0:0:0:0:1"}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -2431,19 +2185,156 @@ func TestInitClusterIP(t *testing.T) { }}, }, }, - expectClusterIP: true, - expectedAllocatedIP: "2000:0:0:0:0:0:0:1", + expectError: false, + enableDualStackAllocator: true, + expectedCountIPs: 1, + expectedClusterIPs: []string{"2000:0:0:0:0:0:0:1"}, }, { - name: "Shouldn't allocate ClusterIP", - families: singleStackIPv4, + name: "Allocate dual stack - on a non dual stack ", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &preferDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: false, + enableDualStackAllocator: false, + expectedCountIPs: 1, + }, + { + name: "Allocate dual stack - upgrade - v4, v6", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &preferDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: false, + enableDualStackAllocator: true, + expectedCountIPs: 2, + }, + { + name: "Allocate dual stack - upgrade - v4, v6 - specific first IP", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &preferDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: false, + enableDualStackAllocator: true, + expectedCountIPs: 2, + expectedClusterIPs: []string{"1.2.3.4"}, + }, + { + name: "Allocate dual stack - upgrade - v6, v4", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &preferDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: false, + enableDualStackAllocator: true, + expectedCountIPs: 2, + }, + { + name: "Allocate dual stack - v4, v6 - specific ips", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &requireDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4", "2000:0:0:0:0:0:0:1"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: false, + enableDualStackAllocator: true, + expectedCountIPs: 2, + expectedClusterIPs: []string{"1.2.3.4", "2000:0:0:0:0:0:0:1"}, + }, + { + name: "Allocate dual stack - upgrade - v6, v4 - specific ips", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &requireDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + ClusterIP: "2000:0:0:0:0:0:0:1", + ClusterIPs: []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: false, + enableDualStackAllocator: true, + expectedCountIPs: 2, + expectedClusterIPs: []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"}, + }, + { + name: "Shouldn't allocate ClusterIP", svc: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, - ClusterIP: api.ClusterIPNone, + ClusterIP: "None", + ClusterIPs: []string{api.ClusterIPNone}, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, @@ -2451,47 +2342,205 @@ func TestInitClusterIP(t *testing.T) { }}, }, }, - expectClusterIP: false, + expectError: false, + enableDualStackAllocator: false, + expectedCountIPs: 0, + }, + { + name: "single stack, ip is pre allocated (ipv4)", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &singleStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + ClusterIP: "1.2.3.4", + ClusterIPs: []string{"1.2.3.4"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: true, + enableDualStackAllocator: false, + expectedCountIPs: 0, + preAllocateClusterIPs: map[api.IPFamily]string{api.IPv4Protocol: "1.2.3.4"}, + }, + + { + name: "single stack, ip is pre allocated (ipv6)", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &singleStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000:0:0:0:0:0:0:1"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: true, + enableDualStackAllocator: true, // ipv6 allocator is always the second one during test + expectedCountIPs: 0, + preAllocateClusterIPs: map[api.IPFamily]string{api.IPv6Protocol: "2000:0:0:0:0:0:0:1"}, + }, + { + name: "Allocate dual stack - upgrade - v6, v4 - specific ips (first ip can't be allocated)", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &requireDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: true, + enableDualStackAllocator: true, + expectedCountIPs: 0, + preAllocateClusterIPs: map[api.IPFamily]string{api.IPv6Protocol: "2000:0:0:0:0:0:0:1"}, + }, + { + name: "Allocate dual stack - upgrade - v6, v4 - specific ips (second ip can't be allocated)", + svc: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + IPFamilyPolicy: &requireDualStack, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + ClusterIP: "2000:0:0:0:0:0:0:1", + ClusterIPs: []string{"2000:0:0:0:0:0:0:1", "1.2.3.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + expectError: true, + enableDualStackAllocator: true, + expectedCountIPs: 0, + preAllocateClusterIPs: map[api.IPFamily]string{api.IPv4Protocol: "1.2.3.4"}, }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() - storage, _, server := NewTestREST(t, nil, tc.families) + // create the rest stack + families := []api.IPFamily{api.IPv4Protocol} + if test.enableDualStackAllocator { + families = append(families, api.IPv6Protocol) + } + storage, _, server := NewTestREST(t, nil, families) defer server.Terminate(t) - allocator := storage.serviceIPs - if tc.useSecondaryAlloc { - allocator = storage.secondaryServiceIPs + copySvc := test.svc.DeepCopy() + + // pre allocate ips if any + for family, ip := range test.preAllocateClusterIPs { + allocator, ok := storage.serviceIPAllocatorsByFamily[family] + if !ok { + t.Fatalf("test is incorrect, allocator does not exist on rest") + } + if err := allocator.Allocate(net.ParseIP(ip)); err != nil { + t.Fatalf("test is incorrect, allocator failed to pre allocate IP with error:%v", err) + } } - allocateSpecificIP := len(tc.svc.Spec.ClusterIP) > 0 && tc.svc.Spec.ClusterIP != "None" - hasAllocatedIP, err := initClusterIP(tc.svc, allocator) + ctx := genericapirequest.NewDefaultContext() + createdSvc, err := storage.Create(ctx, test.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + if test.expectError && err == nil { + t.Fatalf("error was expected, but no error was returned") + } + + if !test.expectError && err != nil { + t.Fatalf("error was not expected, but got error %v", err) + } + if err != nil { - t.Fatalf("unexpected error: %v", err) + return // no more testing needed for this case } + newSvc := createdSvc.(*api.Service) + isValidClusterIPFields(t, storage, copySvc, newSvc) - if hasAllocatedIP != tc.expectClusterIP { - t.Errorf("expected %v, but got %v", tc.expectClusterIP, hasAllocatedIP) - } - - if tc.expectClusterIP { - if !allocator.Has(net.ParseIP(tc.svc.Spec.ClusterIP)) { - t.Errorf("unexpected ClusterIP %q, not allocated", tc.svc.Spec.ClusterIP) + // if it has ips then let us check they have been correctly allocated + if newSvc.Spec.ClusterIPs[0] != api.ClusterIPNone { + for _, ip := range newSvc.Spec.ClusterIPs { + family := api.IPv4Protocol + if netutil.IsIPv6String(ip) { + family = api.IPv6Protocol + } + allocator := storage.serviceIPAllocatorsByFamily[family] + // has retruns true if it was allocated *sigh*.. + if !allocator.Has(net.ParseIP(ip)) { + t.Fatalf("expected ip:%v to be allocated by %v allocator. it was not", ip, family) + } } } - if allocateSpecificIP && tc.expectedAllocatedIP != tc.svc.Spec.ClusterIP { - t.Errorf(" expected ClusterIP %q, but got %q", tc.expectedAllocatedIP, tc.svc.Spec.ClusterIP) + allocatedIPs := 0 + for _, ip := range newSvc.Spec.ClusterIPs { + if ip != api.ClusterIPNone { + allocatedIPs++ + } } + + if allocatedIPs != test.expectedCountIPs { + t.Fatalf("incorrect allocated IP count expected %v got %v", test.expectedCountIPs, allocatedIPs) + } + + for i, ip := range test.expectedClusterIPs { + if i >= len(newSvc.Spec.ClusterIPs) { + t.Fatalf("incorrect ips were assigne. expected to find %+v in %+v", + ip, newSvc.Spec.ClusterIPs) + } + + if ip != newSvc.Spec.ClusterIPs[i] { + t.Fatalf("incorrect ips were assigne. expected to find %+v == %+v at position %v", + ip, newSvc.Spec.ClusterIPs[i], i) + } + } + + // the following apply only on dual stack + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + return + } + + shouldUpgrade := len(newSvc.Spec.IPFamilies) == 2 && *(newSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.serviceIPAllocatorsByFamily) == 2 + if shouldUpgrade && len(newSvc.Spec.ClusterIPs) < 2 { + t.Fatalf("Service should have been upgraded %+v", newSvc) + } + + if !shouldUpgrade && len(newSvc.Spec.ClusterIPs) > 1 { + t.Fatalf("Service should not have been upgraded %+v", newSvc) + } + }) } - } func TestInitNodePorts(t *testing.T) { - storage, _, server := NewTestREST(t, nil, singleStackIPv4) + storage, _, server := NewTestREST(t, nil, []api.IPFamily{api.IPv4Protocol}) defer server.Terminate(t) nodePortOp := portallocator.StartOperation(storage.serviceNodePorts, false) defer nodePortOp.Finish() @@ -2942,252 +2991,2067 @@ func TestUpdateNodePorts(t *testing.T) { } } -func TestAllocGetters(t *testing.T) { - ipv4Service := api.IPv4Protocol - ipv6Service := api.IPv6Protocol +func TestServiceUpgrade(t *testing.T) { + requireDualStack := api.IPFamilyPolicyRequireDualStack + ctx := genericapirequest.NewDefaultContext() testCases := []struct { - name string - - enableDualStack bool - families []api.IPFamily - - expectSpecPrimary bool - expectClusterIPPrimary bool - - svc *api.Service + name string + updateFunc func(svc *api.Service) + enableDualStackAllocator bool + enableDualStackGate bool + allocateIPsBeforeUpdate map[api.IPFamily]string + expectUpgradeError bool + svc api.Service }{ { - name: "spec:v4 ip:v4 dualstack:off", + name: "normal, no upgrade needed", + enableDualStackAllocator: false, + enableDualStackGate: true, + allocateIPsBeforeUpdate: nil, + expectUpgradeError: false, - enableDualStack: false, - families: singleStackIPv4, + updateFunc: func(s *api.Service) { + s.Spec.Selector = map[string]string{"bar": "baz2"} + }, - expectSpecPrimary: true, - expectClusterIPPrimary: true, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, - ClusterIP: "10.0.0.1", + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, }, }, }, { - name: "spec:v4 ip:v4 dualstack:on primary:v4", + name: "error, no upgrade (has single allocator)", + enableDualStackAllocator: false, + enableDualStackGate: true, + allocateIPsBeforeUpdate: nil, + expectUpgradeError: true, - enableDualStack: true, - families: dualStackIPv4Primary, + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol} + }, - expectSpecPrimary: true, - expectClusterIPPrimary: true, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, - ClusterIP: "10.0.0.1", + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + }, + { + name: "upgrade to v4,6", + enableDualStackAllocator: true, + enableDualStackGate: true, + allocateIPsBeforeUpdate: nil, + expectUpgradeError: false, + + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol} + }, + + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + }, + { + name: "upgrade to v4,6 (specific ip)", + enableDualStackAllocator: true, + enableDualStackGate: true, + allocateIPsBeforeUpdate: nil, + expectUpgradeError: false, + + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "2000:0:0:0:0:0:0:1") + s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol} + }, + + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + }, + { + name: "upgrade to v4,6 (specific ip) - fail, ip is not available", + enableDualStackAllocator: true, + enableDualStackGate: true, + allocateIPsBeforeUpdate: map[api.IPFamily]string{api.IPv6Protocol: "2000:0:0:0:0:0:0:1"}, + expectUpgradeError: true, + + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "2000:0:0:0:0:0:0:1") + s.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol} + }, + + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, }, }, }, { - name: "spec:v4 ip:v6 dualstack:on primary:v4", + name: "upgrade to v6,4", + enableDualStackAllocator: true, + enableDualStackGate: true, + allocateIPsBeforeUpdate: nil, + expectUpgradeError: false, - enableDualStack: true, - families: dualStackIPv4Primary, + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol} + }, - expectSpecPrimary: true, - expectClusterIPPrimary: false, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, - ClusterIP: "2000::1", + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, }, }, }, { - name: "spec:v6 ip:v6 dualstack:on primary:v4", + name: "upgrade to v6,4 (specific ip)", + enableDualStackAllocator: true, + enableDualStackGate: true, + allocateIPsBeforeUpdate: nil, + expectUpgradeError: false, - enableDualStack: true, - families: dualStackIPv4Primary, + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "1.2.3.4") + s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol} + }, - expectSpecPrimary: false, - expectClusterIPPrimary: false, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - ClusterIP: "2000::1", + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, }, }, }, { - name: "spec:v6 ip:v4 dualstack:on primary:v4", + name: "upgrade to v6,4 (specific ip) - fail ip is already allocated", + enableDualStackAllocator: true, + enableDualStackGate: true, + allocateIPsBeforeUpdate: map[api.IPFamily]string{api.IPv4Protocol: "1.2.3.4"}, + expectUpgradeError: true, - enableDualStack: true, - families: dualStackIPv4Primary, - - expectSpecPrimary: false, - expectClusterIPPrimary: true, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - ClusterIP: "10.0.0.10", - }, + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &requireDualStack + s.Spec.ClusterIPs = append(s.Spec.ClusterIPs, "1.2.3.4") + s.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol} }, - }, - { - name: "spec:v6 ip:v6 dualstack:off", - - enableDualStack: false, - families: singleStackIPv6, - - expectSpecPrimary: true, - expectClusterIPPrimary: true, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - ClusterIP: "2000::1", - }, - }, - }, - - { - name: "spec:v6 ip:v6 dualstack:on primary:v6", - - enableDualStack: true, - families: dualStackIPv6Primary, - - expectSpecPrimary: true, - expectClusterIPPrimary: true, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - ClusterIP: "2000::1", - }, - }, - }, - - { - name: "spec:v6 ip:v4 dualstack:on primary:v6", - - enableDualStack: true, - families: dualStackIPv6Primary, - - expectSpecPrimary: true, - expectClusterIPPrimary: false, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv6Service, - ClusterIP: "10.0.0.1", - }, - }, - }, - - { - name: "spec:v4 ip:v4 dualstack:on primary:v6", - - enableDualStack: true, - families: dualStackIPv6Primary, - - expectSpecPrimary: false, - expectClusterIPPrimary: false, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, - ClusterIP: "10.0.0.1", - }, - }, - }, - - { - name: "spec:v4 ip:v6 dualstack:on primary:v6", - - enableDualStack: true, - families: dualStackIPv6Primary, - - expectSpecPrimary: false, - expectClusterIPPrimary: true, - - svc: &api.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, - Spec: api.ServiceSpec{ - Selector: map[string]string{"bar": "baz"}, - Type: api.ServiceTypeClusterIP, - IPFamily: &ipv4Service, - ClusterIP: "2000::1", + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, }, }, }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - storage, _, server := NewTestREST(t, nil, tc.families) + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + families := []api.IPFamily{api.IPv4Protocol} + if testCase.enableDualStackAllocator { + families = append(families, api.IPv6Protocol) + } + storage, _, server := NewTestREST(t, nil, families) defer server.Terminate(t) + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, testCase.enableDualStackGate)() - if len(tc.families) == 2 && storage.secondaryServiceIPs == nil { - t.Fatalf("storage must allocate secondary ServiceIPs allocator for dual stack") + obj, err := storage.Create(ctx, &testCase.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + if err != nil { + t.Fatalf("error is unexpected: %v", err) } - alloc := storage.getAllocatorByClusterIP(tc.svc) - if tc.expectClusterIPPrimary { - if !net.IP.Equal(alloc.CIDR().IP, storage.serviceIPs.CIDR().IP) { - t.Fatalf("expected clusterIP primary allocator, but primary allocator was not selected") - } - } else { - if !net.IP.Equal(alloc.CIDR().IP, storage.secondaryServiceIPs.CIDR().IP) { - t.Errorf("expected clusterIP secondary allocator, but secondary allocator was not selected") + createdSvc := obj.(*api.Service) + // allocated IP + for family, ip := range testCase.allocateIPsBeforeUpdate { + alloc := storage.serviceIPAllocatorsByFamily[family] + if err := alloc.Allocate(net.ParseIP(ip)); err != nil { + t.Fatalf("test is incorrect, unable to preallocate ip:%v", ip) } } + // run the modifier + testCase.updateFunc(createdSvc) - alloc = storage.getAllocatorBySpec(tc.svc) - if tc.expectSpecPrimary { - if !net.IP.Equal(alloc.CIDR().IP, storage.serviceIPs.CIDR().IP) { - t.Errorf("expected spec primary allocator, but primary allocator was not selected") - } - } else { - if !net.IP.Equal(alloc.CIDR().IP, storage.secondaryServiceIPs.CIDR().IP) { - t.Errorf("expected spec secondary allocator, but secondary allocator was not selected") + // run the update + updated, _, err := storage.Update(ctx, + createdSvc.Name, + rest.DefaultUpdatedObjectInfo(createdSvc), + rest.ValidateAllObjectFunc, + rest.ValidateAllObjectUpdateFunc, + false, + &metav1.UpdateOptions{}) + + if err != nil && !testCase.expectUpgradeError { + t.Fatalf("an error was not expected during upgrade %v", err) + } + + if err == nil && testCase.expectUpgradeError { + t.Fatalf("error was expected during upgrade") + } + + if err != nil { + return + } + + updatedSvc := updated.(*api.Service) + isValidClusterIPFields(t, storage, updatedSvc, updatedSvc) + + shouldUpgrade := len(createdSvc.Spec.IPFamilies) == 2 && *(createdSvc.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack && len(storage.serviceIPAllocatorsByFamily) == 2 + if shouldUpgrade && len(updatedSvc.Spec.ClusterIPs) < 2 { + t.Fatalf("Service should have been upgraded %+v", createdSvc) + } + + if !shouldUpgrade && len(updatedSvc.Spec.ClusterIPs) > 1 { + t.Fatalf("Service should not have been upgraded %+v", createdSvc) + } + + // make sure that ips were allocated, correctly + for i, family := range updatedSvc.Spec.IPFamilies { + ip := updatedSvc.Spec.ClusterIPs[i] + allocator := storage.serviceIPAllocatorsByFamily[family] + // has retruns true if it was allocated *sigh*.. + if !allocator.Has(net.ParseIP(ip)) { + t.Fatalf("expected ip:%v to be allocated by %v allocator. it was not", ip, family) } } }) } } + +func TestServiceDowngrade(t *testing.T) { + requiredDualStack := api.IPFamilyPolicyRequireDualStack + singleStack := api.IPFamilyPolicySingleStack + ctx := genericapirequest.NewDefaultContext() + testCases := []struct { + name string + updateFunc func(svc *api.Service) + enableDualStackAllocator bool + enableDualStackGate bool + expectDowngradeError bool + svc api.Service + }{ + { + name: "normal, no downgrade needed. single stack => single stack", + enableDualStackAllocator: true, + enableDualStackGate: true, + expectDowngradeError: false, + + updateFunc: func(s *api.Service) { s.Spec.Selector = map[string]string{"bar": "baz2"} }, + + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requiredDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + }, + { + name: "normal, no downgrade needed. dual stack => dual stack", + enableDualStackAllocator: true, + enableDualStackGate: true, + expectDowngradeError: false, + + updateFunc: func(s *api.Service) { s.Spec.Selector = map[string]string{"bar": "baz2"} }, + + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requiredDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + }, + + { + name: "normal, downgrade v4,v6 => v4", + enableDualStackAllocator: true, + enableDualStackGate: true, + expectDowngradeError: false, + + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &singleStack + s.Spec.ClusterIPs = s.Spec.ClusterIPs[0:1] + s.Spec.IPFamilies = s.Spec.IPFamilies[0:1] + }, + + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requiredDualStack, + + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + }, + { + name: "normal, downgrade v6,v4 => v6", + enableDualStackAllocator: true, + enableDualStackGate: true, + expectDowngradeError: false, + + updateFunc: func(s *api.Service) { + s.Spec.IPFamilyPolicy = &singleStack + s.Spec.ClusterIPs = s.Spec.ClusterIPs[0:1] + s.Spec.IPFamilies = s.Spec.IPFamilies[0:1] + }, + + svc: api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Selector: map[string]string{"bar": "baz"}, + SessionAffinity: api.ServiceAffinityNone, + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requiredDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + Ports: []api.ServicePort{{ + Port: 6502, + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(6502), + }}, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + storage, _, server := NewTestREST(t, nil, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}) + defer server.Terminate(t) + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, testCase.enableDualStackGate)() + + obj, err := storage.Create(ctx, &testCase.svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + if err != nil { + t.Fatalf("error is unexpected: %v", err) + } + + createdSvc := obj.(*api.Service) + copySvc := createdSvc.DeepCopy() + + // run the modifier + testCase.updateFunc(createdSvc) + + // run the update + updated, _, err := storage.Update(ctx, + createdSvc.Name, + rest.DefaultUpdatedObjectInfo(createdSvc), + rest.ValidateAllObjectFunc, + rest.ValidateAllObjectUpdateFunc, + false, + &metav1.UpdateOptions{}) + + if err != nil && !testCase.expectDowngradeError { + t.Fatalf("an error was not expected during upgrade %v", err) + } + + if err == nil && testCase.expectDowngradeError { + t.Fatalf("error was expected during upgrade") + } + + if err != nil { + return + } + + updatedSvc := updated.(*api.Service) + isValidClusterIPFields(t, storage, createdSvc, updatedSvc) + + shouldDowngrade := len(copySvc.Spec.ClusterIPs) == 2 && *(createdSvc.Spec.IPFamilyPolicy) == api.IPFamilyPolicySingleStack + + if shouldDowngrade && len(updatedSvc.Spec.ClusterIPs) > 1 { + t.Fatalf("Service should have been downgraded %+v", createdSvc) + } + + if !shouldDowngrade && len(updatedSvc.Spec.ClusterIPs) < 2 { + t.Fatalf("Service should not have been downgraded %+v", createdSvc) + } + + if shouldDowngrade { + releasedIP := copySvc.Spec.ClusterIPs[1] + releasedIPFamily := copySvc.Spec.IPFamilies[1] + allocator := storage.serviceIPAllocatorsByFamily[releasedIPFamily] + + if allocator.Has(net.ParseIP(releasedIP)) { + t.Fatalf("expected ip:%v to be released by %v allocator. it was not", releasedIP, releasedIPFamily) + } + } + }) + } +} + +func TestDefaultingValidation(t *testing.T) { + singleStack := api.IPFamilyPolicySingleStack + preferDualStack := api.IPFamilyPolicyPreferDualStack + requireDualStack := api.IPFamilyPolicyRequireDualStack + + // takes in REST and modify it for a specific config + fnMakeSingleStackIPv4Allocator := func(rest *REST) { + rest.defaultServiceIPFamily = api.IPv4Protocol + rest.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv4Protocol: rest.serviceIPAllocatorsByFamily[api.IPv4Protocol]} + } + + fnMakeSingleStackIPv6Allocator := func(rest *REST) { + rest.defaultServiceIPFamily = api.IPv6Protocol + rest.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{api.IPv6Protocol: rest.serviceIPAllocatorsByFamily[api.IPv6Protocol]} + } + + fnMakeDualStackStackIPv4IPv6Allocator := func(rest *REST) { + // default is v4,v6 rest storage + } + + fnMakeDualStackStackIPv6IPv4Allocator := func(rest *REST) { + rest.defaultServiceIPFamily = api.IPv6Protocol + rest.serviceIPAllocatorsByFamily = map[api.IPFamily]ipallocator.Interface{ + api.IPv6Protocol: rest.serviceIPAllocatorsByFamily[api.IPv6Protocol], + api.IPv4Protocol: rest.serviceIPAllocatorsByFamily[api.IPv4Protocol], + } + } + + testCases := []struct { + name string + modifyRest func(rest *REST) + svc api.Service + + expectedIPFamilyPolicy *api.IPFamilyPolicyType + expectedIPFamilies []api.IPFamily + expectError bool + }{ + //////////////////////////// + // cluster configured as single stack v4 + //////////////////////////// + { + name: "[singlestack:v4] set: externalname on a single stack - v4", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: false, + }, + { + name: "[singlestack:v4] set: nothing", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + + { + name: "[singlestack:v4] set: v4Cluster IPSet", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: v4IPFamilySet", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: v4IPFamilySet", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: PreferDualStack", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: PreferDualStack + v4ClusterIPSet", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: PreferDualStack + v4ClusterIPSet + v4FamilySet", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: v6IPSet", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v4] set: v6IPFamily", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v4] set: RequireDualStack", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v4] set: RequireDualStack + family", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + // selector less + { + name: "[singlestack:v4] set: selectorless, families are ignored", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: selectorless, no families", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: selectorless, user selected", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v4] set: selectorless, user set to preferDualStack", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + // tests incorrect setting for IPFamilyPolicy + { + name: "[singlestack:v4] set: multifamily set to preferDualStack", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v4] set: multifamily set to singleStack", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v4] set: mult clusterips set to preferDualStack", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v4] set: multi clusterips set to singleStack", + modifyRest: fnMakeSingleStackIPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + + //////////////////////////// + // cluster configured as single stack v6 + //////////////////////////// + { + name: "[singlestack:v6] set: externalname on a single stack - v4", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: false, + }, + { + name: "[singlestack:v6] set: nothing", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: v6Cluster IPSet", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: v4IPFamilySet", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: v6IPFamilySet", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000::1"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: PreferDualStack", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: PreferDualStack + v6ClusterIPSet", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: PreferDualStack + v6ClusterIPSet + v6FamilySet", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: v4IPSet", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v6] set: v4IPFamily", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v6] set: RequireDualStack (on single stack ipv6 cluster)", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v6] set: RequireDualStack + family (on single stack ipv6 cluster)", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + + // selector less + { + name: "[singlestack:v6] set: selectorless, families are ignored", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: selectorless, no families", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + ClusterIPs: []string{"None"}, + Type: api.ServiceTypeClusterIP, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: selectorless, user selected", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[singlestack:v6] set: selectorless, user set to preferDualStack", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + // tests incorrect setting for IPFamilyPolicy + { + name: "[singlestack:v6] set: multifamily set to preferDualStack", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v6] set: multifamily set to singleStack", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v6] set: mult clusterips set to preferDualStack", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[singlestack:v6] set: multi clusterips set to singleStack", + modifyRest: fnMakeSingleStackIPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + + //////////////////////////// + // cluster configured as dual stack v4,6 + //////////////////////////// + { + name: "[dualstack:v4,v6] set: externalname on a dual stack - v4,v6", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: nothing", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + + { + name: "[dualstack:v4,v6] set: v4ClusterIPSet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: v4IPFamilySet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: v4IPFamilySet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: v6ClusterIPSet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: v6IPFamilySet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: v6IPFamilySet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000::1"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + // prefer dual stack + { + name: "[dualstack:v4,v6] set: PreferDualStack.", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: PreferDualStack + v4ClusterIPSet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: PreferDualStack + v4ClusterIPSet + v4FamilySet", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + // require dual stack + { + name: "[dualstack:v4,v6] set: RequireDualStack", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + family v4", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + family v6", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + + { + name: "[dualstack:v4,v6] set: RequireDualStack + family +ip v4", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + // + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + family +ip v6", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + ip v6", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + ip v4", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + ips", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + }, + // + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + ips", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1", "10.0.0.10"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: RequireDualStack + ips + families v6,v4", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1", "10.0.0.10"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ips + families v4,v6", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,v6] set: selectorless, no families", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,6] set: selectorless, user selected", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,6] set: selectorless, user set to prefer", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + // tests incorrect setting for IPFamilyPolicy + { + name: "[duakstack:v4,6] set: multifamily set to preferDualStack", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,6] set: multifamily set to singleStack", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[dualstack:v4,6] set: mult clusterips set to preferDualStack", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,6] set: multi clusterips set to singleStack", + modifyRest: fnMakeDualStackStackIPv4IPv6Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + + //////////////////////////// + // cluster configured as dual stack v6,4 + //////////////////////////// + { + name: "[dualstack:v6,v4] set: externalname on a dual stack - v6,v4", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: nothing", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: v4ClusterIPSet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: v4IPFamilySet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + // + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: v4IPFamilySet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: v6ClusterIPSet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: v6IPFamilySet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: v6IPFamilySet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"2000::1"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + // prefer dual stack + { + name: "[dualstack:v6,v4] set: PreferDualStack.", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: PreferDualStack + v4ClusterIPSet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: PreferDualStack + v4ClusterIPSet + v4FamilySet", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + ClusterIPs: []string{"10.0.0.4"}, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + // require dual stack + { + name: "[dualstack:v6,v4] set: RequireDualStack", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + family v4", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + family v6", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + family +ip v4", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + family +ip v6", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ip v6", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ip v4", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ip v4", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ips", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ips", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1", "10.0.0.10"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ips + families v6,v4", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"2000::1", "10.0.0.10"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: RequireDualStack + ips + families v4,v6", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: selectorless, no families", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + }, + }, + expectedIPFamilyPolicy: &requireDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: selectorless, user selected", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &singleStack, + }, + }, + + expectedIPFamilyPolicy: &singleStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,v4] set: selectorless, user set to prefer", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"None"}, + IPFamilyPolicy: &preferDualStack, + }, + }, + + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + expectError: false, + }, + // tests incorrect setting for IPFamilyPolicy + { + name: "[duakstack:v6,5] set: multifamily set to preferDualStack", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v4,6] set: multifamily set to singleStack", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + { + name: "[dualstack:v6,4] set: mult clusterips set to preferDualStack", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &preferDualStack, + }, + }, + expectedIPFamilyPolicy: &preferDualStack, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + expectError: false, + }, + { + name: "[dualstack:v6,4] set: multi clusterips set to singleStack", + modifyRest: fnMakeDualStackStackIPv6IPv4Allocator, + svc: api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{"1.1.1.1", "2001::1"}, + IPFamilies: nil, + IPFamilyPolicy: &singleStack, + }, + }, + expectedIPFamilyPolicy: nil, + expectedIPFamilies: nil, + expectError: true, + }, + } + + // This func only runs when feature gate is on + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + storage, _, server := NewTestREST(t, nil, []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}) // all tests start with dual stack (v4,v6), then modification func takes care of whatever needed + defer server.Terminate(t) + + if testCase.modifyRest != nil { + testCase.modifyRest(storage) + } + + err := storage.tryDefaultValidateServiceClusterIPFields(&testCase.svc) + if err != nil && !testCase.expectError { + t.Fatalf("error %v was not expected", err) + } + + if err == nil && testCase.expectError { + t.Fatalf("error was expected, but no error returned") + } + + if err != nil { + t.Logf("test concluded successfully with terminal error %v", err) + return + } + + // IPFamily Policy + if (testCase.expectedIPFamilyPolicy == nil && testCase.svc.Spec.IPFamilyPolicy != nil) || + (testCase.expectedIPFamilyPolicy != nil && testCase.svc.Spec.IPFamilyPolicy == nil) { + t.Fatalf("ipFamilyPolicy expected:%v got %v", testCase.expectedIPFamilyPolicy, testCase.svc.Spec.IPFamilyPolicy) + } + + if testCase.expectedIPFamilyPolicy != nil { + if *testCase.expectedIPFamilyPolicy != *testCase.svc.Spec.IPFamilyPolicy { + t.Fatalf("ipFamilyPolicy expected:%s got %s", *testCase.expectedIPFamilyPolicy, *testCase.svc.Spec.IPFamilyPolicy) + } + } + + if len(testCase.expectedIPFamilies) != len(testCase.svc.Spec.IPFamilies) { + t.Fatalf("expected len of IPFamilies %v got %v", len(testCase.expectedIPFamilies), len(testCase.svc.Spec.IPFamilies)) + } + + // match families + for i, family := range testCase.expectedIPFamilies { + if testCase.svc.Spec.IPFamilies[i] != family { + t.Fatalf("expected ip family %v at %v got %v", family, i, testCase.svc.Spec.IPFamilies) + } + } + }) + } +} + +// validates that the service created, updated by REST +// has correct ClusterIPs related fields +func isValidClusterIPFields(t *testing.T, storage *REST, pre *api.Service, post *api.Service) { + // valid for gate off/on scenarios + // ClusterIP + if len(post.Spec.ClusterIP) == 0 { + t.Fatalf("service must have clusterIP : %+v", post) + } + // cluster IPs + if len(post.Spec.ClusterIPs) == 0 { + t.Fatalf("new service must have at least one IP: %+v", post) + } + + if post.Spec.ClusterIP != post.Spec.ClusterIPs[0] { + t.Fatalf("clusterIP does not match ClusterIPs[0]: %+v", post) + } + + // if feature gate is not enabled then we need to ignore need fields + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + if post.Spec.IPFamilyPolicy != nil { + t.Fatalf("service must be set to nil for IPFamilyPolicy: %+v", post) + } + + if len(post.Spec.IPFamilies) != 0 { + t.Fatalf("service must be set to nil for IPFamilies: %+v", post) + } + + return + } + + // for gate on scenarios + // prefer dual stack field + if post.Spec.IPFamilyPolicy == nil { + t.Fatalf("service must not have nil for IPFamilyPolicy: %+v", post) + } + + if pre.Spec.IPFamilyPolicy != nil && *(pre.Spec.IPFamilyPolicy) != *(post.Spec.IPFamilyPolicy) { + t.Fatalf("new service must not change PreferDualStack if it was set by user pre: %v post: %v", *(pre.Spec.IPFamilyPolicy), *(post.Spec.IPFamilyPolicy)) + } + + if pre.Spec.IPFamilyPolicy == nil && *(post.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack { + t.Fatalf("new services with prefer dual stack nil must be set to false (prefer dual stack) %+v", post) + } + + // external name or headless services offer no more ClusterIPs field validation + if post.Spec.ClusterIPs[0] == api.ClusterIPNone { + return + } + + // len of ClusteIPs can not be more than Families + // and for providedIPs it needs to match + + // if families are provided then it shouldn't be changed + // this applies on first entry on + if len(pre.Spec.IPFamilies) > 0 { + if len(post.Spec.IPFamilies) == 0 { + t.Fatalf("allocator shouldn't remove ipfamilies[0] pre:%+v, post:%+v", pre.Spec.IPFamilies, post.Spec.IPFamilies) + } + + if pre.Spec.IPFamilies[0] != post.Spec.IPFamilies[0] { + t.Fatalf("allocator shouldn't change post.Spec.IPFamilies[0] pre:%+v post:%+v", pre.Spec.IPFamilies, post.Spec.IPFamilies) + } + } + // if two families are assigned, then they must be dual stack + if len(post.Spec.IPFamilies) == 2 { + if post.Spec.IPFamilies[0] == post.Spec.IPFamilies[1] { + t.Fatalf("allocator assigned two of the same family %+v", post) + } + } + // ips must match families + for i, ip := range post.Spec.ClusterIPs { + isIPv6 := netutil.IsIPv6String(ip) + if isIPv6 && post.Spec.IPFamilies[i] != api.IPv6Protocol { + t.Fatalf("ips does not match assigned families %+v %+v", post.Spec.ClusterIPs, post.Spec.IPFamilies) + } + } +} diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go index 4a1da79cb99..56fc7e04058 100644 --- a/pkg/registry/core/service/storage/storage.go +++ b/pkg/registry/core/service/storage/storage.go @@ -31,10 +31,17 @@ import ( printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/core/service" registry "k8s.io/kubernetes/pkg/registry/core/service" + + netutil "k8s.io/utils/net" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" ) type GenericREST struct { *genericregistry.Store + primaryIPFamily *api.IPFamily + secondaryFamily *api.IPFamily } // NewREST returns a RESTStorage object that will work against services. @@ -61,7 +68,26 @@ func NewGenericREST(optsGetter generic.RESTOptionsGetter, serviceCIDR net.IPNet, statusStore := *store statusStore.UpdateStrategy = service.NewServiceStatusStrategy(strategy) - return &GenericREST{store}, &StatusREST{store: &statusStore}, nil + + ipv4 := api.IPv4Protocol + ipv6 := api.IPv6Protocol + var primaryIPFamily *api.IPFamily = nil + var secondaryFamily *api.IPFamily = nil + if netutil.IsIPv6CIDR(&serviceCIDR) { + primaryIPFamily = &ipv6 + if hasSecondary { + secondaryFamily = &ipv4 + } + } else { + primaryIPFamily = &ipv4 + if hasSecondary { + secondaryFamily = &ipv6 + } + } + genericStore := &GenericREST{store, primaryIPFamily, secondaryFamily} + store.Decorator = genericStore.defaultServiceOnRead // default on read + + return genericStore, &StatusREST{store: &statusStore}, nil } var ( @@ -99,3 +125,112 @@ func (r *StatusREST) Update(ctx context.Context, name string, objInfo rest.Updat // subresources should never allow create on update. return r.store.Update(ctx, name, objInfo, createValidation, updateValidation, false, options) } + +// defaults fields that were not previously set on read. becomes an +// essential part of upgrading a service +func (r *GenericREST) defaultServiceOnRead(obj runtime.Object) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + return nil + } + + service, ok := obj.(*api.Service) + if ok { + return r.defaultAServiceOnRead(service) + } + + serviceList, ok := obj.(*api.ServiceList) + if ok { + return r.defaultServiceList(serviceList) + } + + // this was not an object we can default + return nil +} + +// defaults a service list +func (r *GenericREST) defaultServiceList(serviceList *api.ServiceList) error { + if serviceList == nil { + return nil + } + + for i := range serviceList.Items { + err := r.defaultAServiceOnRead(&serviceList.Items[i]) + if err != nil { + return err + } + } + + return nil +} + +// defaults a single service +func (r *GenericREST) defaultAServiceOnRead(service *api.Service) error { + if service == nil { + return nil + } + + if len(service.Spec.IPFamilies) > 0 { + return nil // already defaulted + } + + // set clusterIPs based on ClusterIP + if len(service.Spec.ClusterIPs) == 0 { + if len(service.Spec.ClusterIP) > 0 { + service.Spec.ClusterIPs = []string{service.Spec.ClusterIP} + } + } + + requireDualStack := api.IPFamilyPolicyRequireDualStack + singleStack := api.IPFamilyPolicySingleStack + preferDualStack := api.IPFamilyPolicyPreferDualStack + // headless services + if len(service.Spec.ClusterIPs) == 1 && service.Spec.ClusterIPs[0] == api.ClusterIPNone { + service.Spec.IPFamilies = []api.IPFamily{*r.primaryIPFamily} + + // headless+selectorless + // headless+selectorless takes both families. Why? + // at this stage we don't know what kind of endpoints (specifically their IPFamilies) the + // user has assigned to this selectorless service. We assume it has dualstack and we default + // it to PreferDualStack on any cluster (single or dualstack configured). + if len(service.Spec.Selector) == 0 { + service.Spec.IPFamilyPolicy = &preferDualStack + if *r.primaryIPFamily == api.IPv4Protocol { + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol) + } else { + service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol) + } + } else { + // headless w/ selector + // this service type follows cluster configuration. this service (selector based) uses a + // selector and will have to follow how the cluster is configured. If the cluster is + // configured to dual stack then the service defaults to PreferDualStack. Otherwise we + // default it to SingleStack. + if r.secondaryFamily != nil { + service.Spec.IPFamilies = append(service.Spec.IPFamilies, *r.secondaryFamily) + service.Spec.IPFamilyPolicy = &preferDualStack + } else { + service.Spec.IPFamilyPolicy = &singleStack + } + } + + } else { + // headful + // make sure a slice exists to receive the families + service.Spec.IPFamilies = make([]api.IPFamily, len(service.Spec.ClusterIPs), len(service.Spec.ClusterIPs)) + for idx, ip := range service.Spec.ClusterIPs { + if netutil.IsIPv6String(ip) { + service.Spec.IPFamilies[idx] = api.IPv6Protocol + } else { + service.Spec.IPFamilies[idx] = api.IPv4Protocol + } + + if len(service.Spec.IPFamilies) == 1 { + service.Spec.IPFamilyPolicy = &singleStack + } else if len(service.Spec.IPFamilies) == 2 { + service.Spec.IPFamilyPolicy = &requireDualStack + } + } + } + + return nil +} diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go index 0189024e946..0fef9c32431 100644 --- a/pkg/registry/core/service/storage/storage_test.go +++ b/pkg/registry/core/service/storage/storage_test.go @@ -17,6 +17,7 @@ limitations under the License. package storage import ( + "net" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,6 +30,10 @@ import ( etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/registry/registrytest" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" ) func newStorage(t *testing.T) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) { @@ -47,6 +52,8 @@ func newStorage(t *testing.T) (*GenericREST, *StatusREST, *etcd3testing.EtcdTest } func validService() *api.Service { + singleStack := api.IPFamilyPolicySingleStack + return &api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", @@ -54,7 +61,10 @@ func validService() *api.Service { }, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, - ClusterIP: "None", + ClusterIP: api.ClusterIPNone, + ClusterIPs: []string{api.ClusterIPNone}, + IPFamilyPolicy: &singleStack, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, SessionAffinity: "None", Type: api.ServiceTypeClusterIP, Ports: []api.ServicePort{{ @@ -84,7 +94,7 @@ func TestCreate(t *testing.T) { &api.Service{ Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, - ClusterIP: "invalid", + ClusterIPs: []string{"invalid"}, SessionAffinity: "None", Type: api.ServiceTypeClusterIP, Ports: []api.ServicePort{{ @@ -110,7 +120,8 @@ func TestUpdate(t *testing.T) { object := obj.(*api.Service) object.Spec = api.ServiceSpec{ Selector: map[string]string{"bar": "baz2"}, - ClusterIP: "None", + ClusterIP: api.ClusterIPNone, + ClusterIPs: []string{api.ClusterIPNone}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeClusterIP, Ports: []api.ServicePort{{ @@ -187,3 +198,278 @@ func TestCategories(t *testing.T) { expected := []string{"all"} registrytest.AssertCategories(t, storage, expected) } + +func makeServiceList() (undefaulted, defaulted *api.ServiceList) { + undefaulted = &api.ServiceList{Items: []api.Service{}} + defaulted = &api.ServiceList{Items: []api.Service{}} + + singleStack := api.IPFamilyPolicySingleStack + requireDualStack := api.IPFamilyPolicyRequireDualStack + + var undefaultedSvc *api.Service + var defaultedSvc *api.Service + + // (for headless) tests must set fields manually according to how the cluster configured + // headless w selector (subject to how the cluster is configured) + undefaultedSvc = &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "headless_with_selector", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{api.ClusterIPNone}, + Selector: map[string]string{"foo": "bar"}, + }, + } + defaultedSvc = undefaultedSvc.DeepCopy() + defaultedSvc.Spec.IPFamilyPolicy = nil // forcing tests to set them + defaultedSvc.Spec.IPFamilies = nil // forcing tests to them + + undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc)) + defaulted.Items = append(defaulted.Items, *(defaultedSvc)) + + // headless w/o selector (always set to require and families according to cluster) + undefaultedSvc = &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "headless_no_selector", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIPs: []string{api.ClusterIPNone}, + Selector: nil, + }, + } + defaultedSvc = undefaultedSvc.DeepCopy() + defaultedSvc.Spec.IPFamilyPolicy = nil // forcing tests to set them + defaultedSvc.Spec.IPFamilies = nil // forcing tests to them + + undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc)) + defaulted.Items = append(defaulted.Items, *(defaultedSvc)) + + // single stack IPv4 + undefaultedSvc = &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "ipv4", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIP: "10.0.0.4", + }, + } + defaultedSvc = undefaultedSvc.DeepCopy() + defaultedSvc.Spec.IPFamilyPolicy = &singleStack + defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol} + + undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc)) + defaulted.Items = append(defaulted.Items, *(defaultedSvc)) + + // single stack IPv6 + undefaultedSvc = &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "ipv6", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIP: "2000::1", + }, + } + defaultedSvc = undefaultedSvc.DeepCopy() + defaultedSvc.Spec.IPFamilyPolicy = &singleStack + defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol} + + undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc)) + defaulted.Items = append(defaulted.Items, *(defaultedSvc)) + + // dualstack IPv4 IPv6 + undefaultedSvc = &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "ipv4_ipv6", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIP: "10.0.0.4", + ClusterIPs: []string{"10.0.0.4", "2000::1"}, + }, + } + defaultedSvc = undefaultedSvc.DeepCopy() + defaultedSvc.Spec.IPFamilyPolicy = &requireDualStack + defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol} + + undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc)) + defaulted.Items = append(defaulted.Items, *(defaultedSvc)) + + // dualstack IPv6 IPv4 + undefaultedSvc = &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "ipv6_ipv4", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1", "10.0.0.4"}, + }, + } + defaultedSvc = undefaultedSvc.DeepCopy() + defaultedSvc.Spec.IPFamilyPolicy = &requireDualStack + defaultedSvc.Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol} + + undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc)) + defaulted.Items = append(defaulted.Items, *(defaultedSvc)) + + // external name + undefaultedSvc = &api.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "external_name", ResourceVersion: "1", Namespace: metav1.NamespaceDefault}, + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + }, + } + + defaultedSvc = undefaultedSvc.DeepCopy() + defaultedSvc.Spec.IPFamilyPolicy = nil + defaultedSvc.Spec.IPFamilies = nil + + undefaulted.Items = append(undefaulted.Items, *(undefaultedSvc)) + defaulted.Items = append(defaulted.Items, *(defaultedSvc)) + + return undefaulted, defaulted +} + +func TestServiceDefaulting(t *testing.T) { + makeStorage := func(t *testing.T, primaryCIDR string, isDualStack bool) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) { + etcdStorage, server := registrytest.NewEtcdStorage(t, "") + restOptions := generic.RESTOptions{ + StorageConfig: etcdStorage, + Decorator: generic.UndecoratedStorage, + DeleteCollectionWorkers: 1, + ResourcePrefix: "services", + } + + _, cidr, err := net.ParseCIDR(primaryCIDR) + if err != nil { + t.Fatalf("failed to parse CIDR %s", primaryCIDR) + } + + serviceStorage, statusStorage, err := NewGenericREST(restOptions, *(cidr), isDualStack) + if err != nil { + t.Fatalf("unexpected error from REST storage: %v", err) + } + return serviceStorage, statusStorage, server + } + + testCases := []struct { + name string + primaryCIDR string + PrimaryIPv6 bool + isDualStack bool + }{ + { + name: "IPv4 single stack cluster", + primaryCIDR: "10.0.0.0/16", + PrimaryIPv6: false, + isDualStack: false, + }, + { + name: "IPv6 single stack cluster", + primaryCIDR: "2000::/108", + PrimaryIPv6: true, + isDualStack: false, + }, + + { + name: "IPv4, IPv6 dual stack cluster", + primaryCIDR: "10.0.0.0/16", + PrimaryIPv6: false, + isDualStack: true, + }, + { + name: "IPv6, IPv4 dual stack cluster", + primaryCIDR: "2000::/108", + PrimaryIPv6: true, + isDualStack: true, + }, + } + + singleStack := api.IPFamilyPolicySingleStack + preferDualStack := api.IPFamilyPolicyPreferDualStack + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + // this func only works with dual stack feature gate on. + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + storage, _, server := makeStorage(t, testCase.primaryCIDR, testCase.isDualStack) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + + undefaultedServiceList, defaultedServiceList := makeServiceList() + // set the two special ones (0: w/ selector, 1: w/o selector) + // review default*OnRead(...) + // Single stack cluster: + // headless w/selector => singlestack + // headless w/o selector => preferDualStack + // dual stack cluster: + // headless w/selector => preferDualStack + // headless w/o selector => preferDualStack + + // assume single stack + defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &singleStack + + // primary family + if testCase.PrimaryIPv6 { + // no selector, gets both families + defaultedServiceList.Items[1].Spec.IPFamilyPolicy = &preferDualStack + defaultedServiceList.Items[1].Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol} + + //assume single stack for w/selector + defaultedServiceList.Items[0].Spec.IPFamilies = []api.IPFamily{api.IPv6Protocol} + // make dualstacked. if needed + if testCase.isDualStack { + defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &preferDualStack + defaultedServiceList.Items[0].Spec.IPFamilies = append(defaultedServiceList.Items[0].Spec.IPFamilies, api.IPv4Protocol) + } + } else { + // no selector gets both families + defaultedServiceList.Items[1].Spec.IPFamilyPolicy = &preferDualStack + defaultedServiceList.Items[1].Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol} + + // assume single stack for w/selector + defaultedServiceList.Items[0].Spec.IPFamilies = []api.IPFamily{api.IPv4Protocol} + // make dualstacked. if needed + if testCase.isDualStack { + defaultedServiceList.Items[0].Spec.IPFamilyPolicy = &preferDualStack + defaultedServiceList.Items[0].Spec.IPFamilies = append(defaultedServiceList.Items[0].Spec.IPFamilies, api.IPv6Protocol) + } + } + + // data is now ready for testing over various cluster configuration + compareSvc := func(out api.Service, expected api.Service) { + if expected.Spec.IPFamilyPolicy == nil && out.Spec.IPFamilyPolicy != nil { + t.Fatalf("service %+v expected IPFamilyPolicy to be nil", out) + } + if expected.Spec.IPFamilyPolicy != nil && out.Spec.IPFamilyPolicy == nil { + t.Fatalf("service %+v expected IPFamilyPolicy not to be nil", out) + } + + if expected.Spec.IPFamilyPolicy != nil { + if *out.Spec.IPFamilyPolicy != *expected.Spec.IPFamilyPolicy { + t.Fatalf("service %+v expected IPFamilyPolicy %v got %v", out, *expected.Spec.IPFamilyPolicy, *out.Spec.IPFamilyPolicy) + } + } + + if len(out.Spec.IPFamilies) != len(expected.Spec.IPFamilies) { + t.Fatalf("service %+v expected len(IPFamilies) == %v", out, len(expected.Spec.IPFamilies)) + } + for i, ipfamily := range out.Spec.IPFamilies { + if expected.Spec.IPFamilies[i] != ipfamily { + t.Fatalf("service %+v expected ip families %+v", out, expected.Spec.IPFamilies) + } + } + } + + copyUndefaultedList := undefaultedServiceList.DeepCopy() + // run for each service + for i, svc := range copyUndefaultedList.Items { + storage.defaultServiceOnRead(&svc) + compareSvc(svc, defaultedServiceList.Items[i]) + } + + copyUndefaultedList = undefaultedServiceList.DeepCopy() + // run as a servicr list + storage.defaultServiceOnRead(copyUndefaultedList) + for i, svc := range copyUndefaultedList.Items { + compareSvc(svc, defaultedServiceList.Items[i]) + } + + // if there are more tests needed then the last call need to work + // with copy of undefaulted list since + }) + } +} diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go index 4e160b3cf04..0c59741e4cc 100644 --- a/pkg/registry/core/service/strategy.go +++ b/pkg/registry/core/service/strategy.go @@ -95,12 +95,8 @@ func (strategy svcStrategy) PrepareForCreate(ctx context.Context, obj runtime.Ob service := obj.(*api.Service) service.Status = api.ServiceStatus{} - if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && service.Spec.IPFamily == nil { - family := strategy.ipFamilies[0] - service.Spec.IPFamily = &family - } - - dropServiceDisabledFields(service, nil) + normalizeClusterIPs(nil, service) + strategy.dropServiceDisabledFields(service, nil) } // PrepareForUpdate sets contextual defaults and clears fields that are not allowed to be set by end users on update. @@ -109,23 +105,18 @@ func (strategy svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runti oldService := old.(*api.Service) newService.Status = oldService.Status - if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && newService.Spec.IPFamily == nil { - if oldService.Spec.IPFamily != nil { - newService.Spec.IPFamily = oldService.Spec.IPFamily - } else { - family := strategy.ipFamilies[0] - newService.Spec.IPFamily = &family - } - } - - dropServiceDisabledFields(newService, oldService) + normalizeClusterIPs(oldService, newService) + strategy.dropServiceDisabledFields(newService, oldService) + // if service was converted from ClusterIP => ExternalName + // then clear ClusterIPs, IPFamilyPolicy and IPFamilies + clearClusterIPRelatedFields(newService, oldService) + trimFieldsForDualStackDowngrade(newService, oldService) } // Validate validates a new service. func (strategy svcStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { service := obj.(*api.Service) allErrs := validation.ValidateServiceCreate(service) - allErrs = append(allErrs, validation.ValidateConditionalService(service, nil, strategy.ipFamilies)...) return allErrs } @@ -139,7 +130,6 @@ func (svcStrategy) AllowCreateOnUpdate() bool { func (strategy svcStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { allErrs := validation.ValidateServiceUpdate(obj.(*api.Service), old.(*api.Service)) - allErrs = append(allErrs, validation.ValidateConditionalService(obj.(*api.Service), old.(*api.Service), strategy.ipFamilies)...) return allErrs } @@ -158,8 +148,10 @@ func (svcStrategy) Export(ctx context.Context, obj runtime.Object, exact bool) e if exact { return nil } - if t.Spec.ClusterIP != api.ClusterIPNone { + //set ClusterIPs as nil - if ClusterIPs[0] != None + if len(t.Spec.ClusterIPs) > 0 && t.Spec.ClusterIPs[0] != api.ClusterIPNone { t.Spec.ClusterIP = "" + t.Spec.ClusterIPs = nil } if t.Spec.Type == api.ServiceTypeNodePort { for i := range t.Spec.Ports { @@ -174,10 +166,13 @@ func (svcStrategy) Export(ctx context.Context, obj runtime.Object, exact bool) e // if !utilfeature.DefaultFeatureGate.Enabled(features.MyFeature) && !myFeatureInUse(oldSvc) { // newSvc.Spec.MyFeature = nil // } -func dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) { - // Drop IPFamily if DualStack is not enabled - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && !serviceIPFamilyInUse(oldSvc) { - newSvc.Spec.IPFamily = nil +func (strategy svcStrategy) dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) { + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && !strategy.serviceDualStackFieldsInUse(oldSvc) { + newSvc.Spec.IPFamilies = nil + newSvc.Spec.IPFamilyPolicy = nil + if len(newSvc.Spec.ClusterIPs) > 1 { + newSvc.Spec.ClusterIPs = newSvc.Spec.ClusterIPs[0:1] + } } // Drop TopologyKeys if ServiceTopology is not enabled @@ -187,14 +182,16 @@ func dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) { } // returns true if svc.Spec.ServiceIPFamily field is in use -func serviceIPFamilyInUse(svc *api.Service) bool { +func (strategy svcStrategy) serviceDualStackFieldsInUse(svc *api.Service) bool { if svc == nil { return false } - if svc.Spec.IPFamily != nil { - return true - } - return false + + ipFamilyPolicyInUse := svc.Spec.IPFamilyPolicy != nil + ipFamiliesInUse := len(svc.Spec.IPFamilies) > 0 + ClusterIPsInUse := len(svc.Spec.ClusterIPs) > 1 + + return ipFamilyPolicyInUse || ipFamiliesInUse || ClusterIPsInUse } // returns true if svc.Spec.TopologyKeys field is in use @@ -226,3 +223,126 @@ func (serviceStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runt func (serviceStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { return validation.ValidateServiceStatusUpdate(obj.(*api.Service), old.(*api.Service)) } + +// normalizeClusterIPs adjust clusterIPs based on ClusterIP +func normalizeClusterIPs(oldSvc *api.Service, newSvc *api.Service) { + // In all cases here, we don't need to over-think the inputs. Validation + // will be called on the new object soon enough. All this needs to do is + // try to divine what user meant with these linked fields. The below + // is verbosely written for clarity. + + // **** IMPORTANT ***** + // as a governing rule. User must (either) + // -- Use singular only (old client) + // -- singular and plural fields (new clients) + + if oldSvc == nil { + // This was a create operation. + // User specified singular and not plural (e.g. an old client), so init + // plural for them. + if len(newSvc.Spec.ClusterIP) > 0 && len(newSvc.Spec.ClusterIPs) == 0 { + newSvc.Spec.ClusterIPs = []string{newSvc.Spec.ClusterIP} + return + } + + // we don't init singular based on plural because + // new client must use both fields + + // Either both were not specified (will be allocated) or both were + // specified (will be validated). + return + } + + // This was an update operation + + // ClusterIPs were cleared by an old client which was trying to patch + // some field and didn't provide ClusterIPs + if len(oldSvc.Spec.ClusterIPs) > 0 && len(newSvc.Spec.ClusterIPs) == 0 { + // if ClusterIP is the same, then it is an old client trying to + // patch service and didn't provide ClusterIPs + if oldSvc.Spec.ClusterIP == newSvc.Spec.ClusterIP { + newSvc.Spec.ClusterIPs = oldSvc.Spec.ClusterIPs + } + } + + // clusterIP is not the same + if oldSvc.Spec.ClusterIP != newSvc.Spec.ClusterIP { + // this is a client trying to clear it + if len(oldSvc.Spec.ClusterIP) > 0 && len(newSvc.Spec.ClusterIP) == 0 { + // if clusterIPs are the same, then clear on their behalf + if sameStringSlice(oldSvc.Spec.ClusterIPs, newSvc.Spec.ClusterIPs) { + newSvc.Spec.ClusterIPs = nil + } + + // if they provided nil, then we are fine (handled by patching case above) + // if they changed it then validation will catch it + } else { + // ClusterIP has changed but not cleared *and* ClusterIPs are the same + // then we set ClusterIPs based on ClusterIP + if sameStringSlice(oldSvc.Spec.ClusterIPs, newSvc.Spec.ClusterIPs) { + newSvc.Spec.ClusterIPs = []string{newSvc.Spec.ClusterIP} + } + } + } +} + +func sameStringSlice(a []string, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// clearClusterIPRelatedFields ensures a backward compatible behavior when the user uses +// an older client to convert a service from ClusterIP to ExternalName. We do that by removing +// the newly introduced fields. +func clearClusterIPRelatedFields(newService, oldService *api.Service) { + if newService.Spec.Type == api.ServiceTypeExternalName && oldService.Spec.Type != api.ServiceTypeExternalName { + // IMPORTANT: this function is always called AFTER ClusterIPs normalization + // which clears ClusterIPs according to ClusterIP. The below checks for ClusterIP + clusterIPReset := len(newService.Spec.ClusterIP) == 0 && len(oldService.Spec.ClusterIP) > 0 + + if clusterIPReset { + // reset other fields + newService.Spec.ClusterIP = "" + newService.Spec.ClusterIPs = nil + newService.Spec.IPFamilies = nil + newService.Spec.IPFamilyPolicy = nil + } + } +} + +// this func allows user to downgrade a service by just changing +// IPFamilyPolicy to SingleStack +func trimFieldsForDualStackDowngrade(newService, oldService *api.Service) { + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + return + } + + // not an update + if oldService == nil { + return + } + + oldIsDualStack := oldService.Spec.IPFamilyPolicy != nil && + (*oldService.Spec.IPFamilyPolicy == api.IPFamilyPolicyRequireDualStack || + *oldService.Spec.IPFamilyPolicy == api.IPFamilyPolicyPreferDualStack) + + newIsNotDualStack := newService.Spec.IPFamilyPolicy != nil && *newService.Spec.IPFamilyPolicy == api.IPFamilyPolicySingleStack + + // if user want to downgrade then we auto remove secondary ip and family + if oldIsDualStack && newIsNotDualStack { + if len(newService.Spec.ClusterIPs) > 1 { + newService.Spec.ClusterIPs = newService.Spec.ClusterIPs[0:1] + } + + if len(newService.Spec.IPFamilies) > 1 { + newService.Spec.IPFamilies = newService.Spec.IPFamilies[0:1] + } + } +} diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go index 630a08d9c8b..843b386e4d9 100644 --- a/pkg/registry/core/service/strategy_test.go +++ b/pkg/registry/core/service/strategy_test.go @@ -83,7 +83,7 @@ func TestExportService(t *testing.T) { Namespace: "bar", }, Spec: api.ServiceSpec{ - ClusterIP: "10.0.0.1", + ClusterIPs: []string{"10.0.0.1"}, }, Status: api.ServiceStatus{ LoadBalancer: api.LoadBalancerStatus{ @@ -99,10 +99,38 @@ func TestExportService(t *testing.T) { Namespace: "bar", }, Spec: api.ServiceSpec{ - ClusterIP: "", + ClusterIPs: nil, }, }, }, + { + objIn: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + Spec: api.ServiceSpec{ + ClusterIPs: []string{"10.0.0.1", "2001::1"}, + }, + Status: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{ + {IP: "1.2.3.4"}, + }, + }, + }, + }, + objOut: &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + Spec: api.ServiceSpec{ + ClusterIPs: nil, + }, + }, + }, + { objIn: &api.Pod{}, expectErr: true, @@ -146,7 +174,6 @@ func TestCheckGeneratedNameError(t *testing.T) { } func makeValidService() api.Service { - defaultServiceIPFamily := api.IPv4Protocol return api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "valid", @@ -160,137 +187,16 @@ func makeValidService() api.Service { SessionAffinity: "None", Type: api.ServiceTypeClusterIP, Ports: []api.ServicePort{{Name: "p", Protocol: "TCP", Port: 8675, TargetPort: intstr.FromInt(8675)}}, - IPFamily: &defaultServiceIPFamily, }, } } // TODO: This should be done on types that are not part of our API -func TestBeforeCreate(t *testing.T) { - withIP := func(family *api.IPFamily, ip string) *api.Service { - svc := makeValidService() - svc.Spec.IPFamily = family - svc.Spec.ClusterIP = ip - return &svc - } - - ipv4 := api.IPv4Protocol - ipv6 := api.IPv6Protocol - testCases := []struct { - name string - cidr string - configureDualStack bool - enableDualStack bool - in *api.Service - expect *api.Service - expectErr bool - }{ - { - name: "does not set ipfamily when dual stack gate is disabled", - cidr: "10.0.0.0/16", - in: withIP(nil, ""), - expect: withIP(nil, ""), - }, - - { - name: "clears ipfamily when dual stack gate is disabled", - cidr: "10.0.0.0/16", - in: withIP(&ipv4, ""), - expect: withIP(nil, ""), - }, - - { - name: "allows ipfamily to configured ipv4 value", - cidr: "10.0.0.0/16", - enableDualStack: true, - in: withIP(nil, ""), - expect: withIP(&ipv4, ""), - }, - { - name: "allows ipfamily to configured ipv4 value when dual stack is in use", - cidr: "10.0.0.0/16", - enableDualStack: true, - configureDualStack: true, - in: withIP(nil, ""), - expect: withIP(&ipv4, ""), - }, - { - name: "allows ipfamily to configured ipv6 value", - cidr: "fd00::/64", - enableDualStack: true, - in: withIP(nil, ""), - expect: withIP(&ipv6, ""), - }, - { - name: "allows ipfamily to configured ipv6 value when dual stack is in use", - cidr: "fd00::/64", - enableDualStack: true, - configureDualStack: true, - in: withIP(nil, ""), - expect: withIP(&ipv6, ""), - }, - - { - name: "rejects ipv6 ipfamily when single-stack ipv4", - enableDualStack: true, - cidr: "10.0.0.0/16", - in: withIP(&ipv6, ""), - expectErr: true, - }, - { - name: "rejects ipv4 ipfamily when single-stack ipv6", - enableDualStack: true, - cidr: "fd00::/64", - in: withIP(&ipv4, ""), - expectErr: true, - }, - { - name: "rejects implicit ipv4 ipfamily when single-stack ipv6", - enableDualStack: true, - cidr: "fd00::/64", - in: withIP(nil, "10.0.1.0"), - expectErr: true, - }, - { - name: "rejects implicit ipv6 ipfamily when single-stack ipv4", - enableDualStack: true, - cidr: "10.0.0.0/16", - in: withIP(nil, "fd00::1"), - expectErr: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - testStrategy, _ := newStrategy(tc.cidr, tc.configureDualStack) - ctx := genericapirequest.NewDefaultContext() - err := rest.BeforeCreate(testStrategy, ctx, runtime.Object(tc.in)) - if tc.expectErr != (err != nil) { - t.Fatalf("unexpected error: %v", err) - } - if err != nil { - return - } - if tc.expect != nil && tc.in != nil { - tc.expect.ObjectMeta = tc.in.ObjectMeta - } - if !reflect.DeepEqual(tc.expect, tc.in) { - t.Fatalf("unexpected change: %s", diff.ObjectReflectDiff(tc.expect, tc.in)) - } - }) - } -} - func TestBeforeUpdate(t *testing.T) { testCases := []struct { - name string - enableDualStack bool - defaultIPv6 bool - allowSecondary bool - tweakSvc func(oldSvc, newSvc *api.Service) // given basic valid services, each test case can customize them - expectErr bool - expectObj func(t *testing.T, svc *api.Service) + name string + tweakSvc func(oldSvc, newSvc *api.Service) // given basic valid services, each test case can customize them + expectErr bool }{ { name: "no change", @@ -323,24 +229,11 @@ func TestBeforeUpdate(t *testing.T) { { name: "change ClusterIP", tweakSvc: func(oldSvc, newSvc *api.Service) { - oldSvc.Spec.ClusterIP = "1.2.3.4" - newSvc.Spec.ClusterIP = "4.3.2.1" + oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} + newSvc.Spec.ClusterIPs = []string{"4.3.2.1"} }, expectErr: true, }, - { - name: "clear IP family is allowed (defaulted back by before update)", - enableDualStack: true, - tweakSvc: func(oldSvc, newSvc *api.Service) { - oldSvc.Spec.IPFamily = nil - }, - expectErr: false, - expectObj: func(t *testing.T, svc *api.Service) { - if svc.Spec.IPFamily == nil { - t.Errorf("ipfamily was not defaulted") - } - }, - }, { name: "change selector", tweakSvc: func(oldSvc, newSvc *api.Service) { @@ -351,32 +244,22 @@ func TestBeforeUpdate(t *testing.T) { } for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - var cidr string - if tc.defaultIPv6 { - cidr = "ffd0::/64" - } else { - cidr = "172.30.0.0/16" - } - strategy, _ := newStrategy(cidr, tc.allowSecondary) - oldSvc := makeValidService() - newSvc := makeValidService() - tc.tweakSvc(&oldSvc, &newSvc) - ctx := genericapirequest.NewDefaultContext() - err := rest.BeforeUpdate(strategy, ctx, runtime.Object(&newSvc), runtime.Object(&oldSvc)) - if tc.expectObj != nil { - tc.expectObj(t, &newSvc) - } - if tc.expectErr && err == nil { - t.Fatalf("unexpected non-error: %v", err) - } - if !tc.expectErr && err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) + strategy, _ := newStrategy("172.30.0.0/16", false) + + oldSvc := makeValidService() + newSvc := makeValidService() + tc.tweakSvc(&oldSvc, &newSvc) + ctx := genericapirequest.NewDefaultContext() + err := rest.BeforeUpdate(strategy, ctx, runtime.Object(&oldSvc), runtime.Object(&newSvc)) + if tc.expectErr && err == nil { + t.Errorf("unexpected non-error for %q", tc.name) + } + if !tc.expectErr && err != nil { + t.Errorf("unexpected error for %q: %v", tc.name, err) + } } } + func TestServiceStatusStrategy(t *testing.T) { _, testStatusStrategy := newStrategy("10.0.0.0/16", false) ctx := genericapirequest.NewDefaultContext() @@ -408,16 +291,20 @@ func TestServiceStatusStrategy(t *testing.T) { } } -func makeServiceWithIPFamily(ipFamily *api.IPFamily) *api.Service { +func makeServiceWithIPFamilies(ipfamilies []api.IPFamily, ipFamilyPolicy *api.IPFamilyPolicyType) *api.Service { return &api.Service{ Spec: api.ServiceSpec{ - IPFamily: ipFamily, + IPFamilies: ipfamilies, + IPFamilyPolicy: ipFamilyPolicy, }, } } + func TestDropDisabledField(t *testing.T) { - ipv4Service := api.IPv4Protocol - ipv6Service := api.IPv6Protocol + requireDualStack := api.IPFamilyPolicyRequireDualStack + preferDualStack := api.IPFamilyPolicyPreferDualStack + singleStack := api.IPFamilyPolicySingleStack + testCases := []struct { name string enableDualStack bool @@ -428,44 +315,60 @@ func TestDropDisabledField(t *testing.T) { { name: "not dual stack, field not used", enableDualStack: false, - svc: makeServiceWithIPFamily(nil), + svc: makeServiceWithIPFamilies(nil, nil), oldSvc: nil, - compareSvc: makeServiceWithIPFamily(nil), - }, - { - name: "not dual stack, field used in new, not in old", - enableDualStack: false, - svc: makeServiceWithIPFamily(&ipv4Service), - oldSvc: nil, - compareSvc: makeServiceWithIPFamily(nil), + compareSvc: makeServiceWithIPFamilies(nil, nil), }, { name: "not dual stack, field used in old and new", enableDualStack: false, - svc: makeServiceWithIPFamily(&ipv4Service), - oldSvc: makeServiceWithIPFamily(&ipv4Service), - compareSvc: makeServiceWithIPFamily(&ipv4Service), + svc: makeServiceWithIPFamilies([]api.IPFamily{api.IPv4Protocol}, nil), + oldSvc: makeServiceWithIPFamilies([]api.IPFamily{api.IPv4Protocol}, nil), + compareSvc: makeServiceWithIPFamilies([]api.IPFamily{api.IPv4Protocol}, nil), }, { name: "dualstack, field used", enableDualStack: true, - svc: makeServiceWithIPFamily(&ipv6Service), + svc: makeServiceWithIPFamilies([]api.IPFamily{api.IPv6Protocol}, nil), oldSvc: nil, - compareSvc: makeServiceWithIPFamily(&ipv6Service), + compareSvc: makeServiceWithIPFamilies([]api.IPFamily{api.IPv6Protocol}, nil), + }, + /* preferDualStack field */ + { + name: "not dual stack, fields is not use", + enableDualStack: false, + svc: makeServiceWithIPFamilies(nil, nil), + oldSvc: nil, + compareSvc: makeServiceWithIPFamilies(nil, nil), }, { - name: "dualstack, field used, changed", - enableDualStack: true, - svc: makeServiceWithIPFamily(&ipv6Service), - oldSvc: makeServiceWithIPFamily(&ipv4Service), - compareSvc: makeServiceWithIPFamily(&ipv6Service), + name: "not dual stack, fields used in new, not in old", + enableDualStack: false, + svc: makeServiceWithIPFamilies(nil, &preferDualStack), + oldSvc: nil, + compareSvc: makeServiceWithIPFamilies(nil, nil), }, { - name: "dualstack, field used, not changed", + name: "not dual stack, fields used in new, not in old", + enableDualStack: false, + svc: makeServiceWithIPFamilies(nil, &requireDualStack), + oldSvc: nil, + compareSvc: makeServiceWithIPFamilies(nil, nil), + }, + + { + name: "not dual stack, fields not used in old (single stack)", + enableDualStack: false, + svc: makeServiceWithIPFamilies(nil, nil), + oldSvc: makeServiceWithIPFamilies(nil, &singleStack), + compareSvc: makeServiceWithIPFamilies(nil, nil), + }, + { + name: "dualstack, field used", enableDualStack: true, - svc: makeServiceWithIPFamily(&ipv6Service), - oldSvc: makeServiceWithIPFamily(&ipv6Service), - compareSvc: makeServiceWithIPFamily(&ipv6Service), + svc: makeServiceWithIPFamilies(nil, &singleStack), + oldSvc: nil, + compareSvc: makeServiceWithIPFamilies(nil, &singleStack), }, /* add more tests for other dropped fields as needed */ @@ -474,7 +377,10 @@ func TestDropDisabledField(t *testing.T) { func() { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() old := tc.oldSvc.DeepCopy() - dropServiceDisabledFields(tc.svc, tc.oldSvc) + + // to test against user using IPFamily not set on cluster + svcStrategy := svcStrategy{ipFamilies: []api.IPFamily{api.IPv4Protocol}} + svcStrategy.dropServiceDisabledFields(tc.svc, tc.oldSvc) // old node should never be changed if !reflect.DeepEqual(tc.oldSvc, old) { @@ -488,3 +394,612 @@ func TestDropDisabledField(t *testing.T) { } } + +func TestNormalizeClusterIPs(t *testing.T) { + testCases := []struct { + name string + oldService *api.Service + newService *api.Service + expectedClusterIP string + expectedClusterIPs []string + }{ + + { + name: "new - only clusterip used", + oldService: nil, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: nil, + }, + }, + expectedClusterIP: "10.0.0.10", + expectedClusterIPs: []string{"10.0.0.10"}, + }, + + { + name: "new - only clusterips used", + oldService: nil, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedClusterIP: "", // this is a validation issue, and validation will catch it + expectedClusterIPs: []string{"10.0.0.10"}, + }, + + { + name: "new - both used", + oldService: nil, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedClusterIP: "10.0.0.10", + expectedClusterIPs: []string{"10.0.0.10"}, + }, + + { + name: "update - no change", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedClusterIP: "10.0.0.10", + expectedClusterIPs: []string{"10.0.0.10"}, + }, + + { + name: "update - malformed change", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.11", + ClusterIPs: []string{"10.0.0.11"}, + }, + }, + expectedClusterIP: "10.0.0.11", + expectedClusterIPs: []string{"10.0.0.11"}, + }, + + { + name: "update - malformed change on secondary ip", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.11", + ClusterIPs: []string{"10.0.0.11", "3000::1"}, + }, + }, + expectedClusterIP: "10.0.0.11", + expectedClusterIPs: []string{"10.0.0.11", "3000::1"}, + }, + + { + name: "update - upgrade", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + }, + expectedClusterIP: "10.0.0.10", + expectedClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + { + name: "update - downgrade", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedClusterIP: "10.0.0.10", + expectedClusterIPs: []string{"10.0.0.10"}, + }, + + { + name: "update - user cleared cluster IP", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedClusterIP: "", + expectedClusterIPs: nil, + }, + + { + name: "update - user cleared clusterIPs", // *MUST* REMAIN FOR OLD CLIENTS + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: nil, + }, + }, + expectedClusterIP: "10.0.0.10", + expectedClusterIPs: []string{"10.0.0.10"}, + }, + + { + name: "update - user cleared both", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "", + ClusterIPs: nil, + }, + }, + expectedClusterIP: "", + expectedClusterIPs: nil, + }, + + { + name: "update - user cleared ClusterIP but changed clusterIPs", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "", + ClusterIPs: []string{"10.0.0.11"}, + }, + }, + expectedClusterIP: "", /* validation catches this */ + expectedClusterIPs: []string{"10.0.0.11"}, + }, + + { + name: "update - user cleared ClusterIPs but changed ClusterIP", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.11", + ClusterIPs: nil, + }, + }, + expectedClusterIP: "10.0.0.11", + expectedClusterIPs: nil, + }, + + { + name: "update - user changed from None to ClusterIP", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "None", + ClusterIPs: []string{"None"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"None"}, + }, + }, + expectedClusterIP: "10.0.0.10", + expectedClusterIPs: []string{"10.0.0.10"}, + }, + + { + name: "update - user changed from ClusterIP to None", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "None", + ClusterIPs: []string{"10.0.0.10"}, + }, + }, + expectedClusterIP: "None", + expectedClusterIPs: []string{"None"}, + }, + + { + name: "update - user changed from ClusterIP to None and changed ClusterIPs in a dual stack (new client making a mistake)", + oldService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.10", + ClusterIPs: []string{"10.0.0.10", "2000::1"}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + ClusterIP: "None", + ClusterIPs: []string{"10.0.0.11", "2000::1"}, + }, + }, + expectedClusterIP: "None", + expectedClusterIPs: []string{"10.0.0.11", "2000::1"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + normalizeClusterIPs(tc.oldService, tc.newService) + + if tc.newService == nil { + t.Fatalf("unexpected new service to be nil") + } + + if tc.newService.Spec.ClusterIP != tc.expectedClusterIP { + t.Fatalf("expected clusterIP [%v] got [%v]", tc.expectedClusterIP, tc.newService.Spec.ClusterIP) + } + + if len(tc.newService.Spec.ClusterIPs) != len(tc.expectedClusterIPs) { + t.Fatalf("expected clusterIPs %v got %v", tc.expectedClusterIPs, tc.newService.Spec.ClusterIPs) + } + + for idx, clusterIP := range tc.newService.Spec.ClusterIPs { + if clusterIP != tc.expectedClusterIPs[idx] { + t.Fatalf("expected clusterIP [%v] at index[%v] got [%v]", tc.expectedClusterIPs[idx], idx, tc.newService.Spec.ClusterIPs[idx]) + + } + } + }) + } + +} + +func TestClearClusterIPRelatedFields(t *testing.T) { + // + // NOTE the data fed to this test assums that ClusterIPs normalization is + // already done check PrepareFor*(..) strategy + // + singleStack := api.IPFamilyPolicySingleStack + requireDualStack := api.IPFamilyPolicyRequireDualStack + testCases := []struct { + name string + oldService *api.Service + newService *api.Service + shouldClear bool + }{ + { + name: "should clear, single stack converting to external name", + shouldClear: true, + + oldService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &singleStack, + ClusterIP: "10.0.0.4", + ClusterIPs: []string{"10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + IPFamilyPolicy: &singleStack, + ClusterIP: "", + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + }, + + { + name: "should clear, dual stack converting to external name(normalization removed all ips)", + shouldClear: true, + + oldService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1", "10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + IPFamilyPolicy: &singleStack, + ClusterIP: "", + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + }, + + { + name: "should NOT clear, single stack converting to external name ClusterIPs was not cleared", + shouldClear: false, + + oldService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &singleStack, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + IPFamilyPolicy: &singleStack, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + }, + + { + name: "should NOT clear, dualstack cleared primary and changed ClusterIPs", + shouldClear: true, + + oldService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1", "10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + IPFamilyPolicy: &singleStack, + ClusterIP: "", + ClusterIPs: []string{"2000::1", "10.0.0.5"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + }, + { + name: "should clear, dualstack user removed ClusterIPs", + shouldClear: true, + + oldService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + IPFamilyPolicy: &requireDualStack, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1", "10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + + newService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeExternalName, + IPFamilyPolicy: &singleStack, + ClusterIP: "", + ClusterIPs: nil, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + }, + { + name: "should NOT clear, dualstack service changing selector", + shouldClear: false, + + oldService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + Selector: map[string]string{"foo": "bar"}, + IPFamilyPolicy: &requireDualStack, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1", "10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + }, + }, + newService: &api.Service{ + Spec: api.ServiceSpec{ + Type: api.ServiceTypeClusterIP, + Selector: map[string]string{"foo": "baz"}, + IPFamilyPolicy: &singleStack, + ClusterIP: "2000::1", + ClusterIPs: []string{"2000::1", "10.0.0.4"}, + IPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + clearClusterIPRelatedFields(testCase.newService, testCase.oldService) + + if testCase.shouldClear && len(testCase.newService.Spec.ClusterIPs) != 0 { + t.Fatalf("expected clusterIPs to be cleared") + } + + if testCase.shouldClear && len(testCase.newService.Spec.IPFamilies) != 0 { + t.Fatalf("expected ipfamilies to be cleared") + } + + if testCase.shouldClear && testCase.newService.Spec.IPFamilyPolicy != nil { + t.Fatalf("expected ipfamilypolicy to be cleared") + } + + if !testCase.shouldClear && len(testCase.newService.Spec.ClusterIPs) == 0 { + t.Fatalf("expected clusterIPs NOT to be cleared") + } + + if !testCase.shouldClear && len(testCase.newService.Spec.IPFamilies) == 0 { + t.Fatalf("expected ipfamilies NOT to be cleared") + } + + if !testCase.shouldClear && testCase.newService.Spec.IPFamilyPolicy == nil { + t.Fatalf("expected ipfamilypolicy NOT to be cleared") + } + + }) + } +} + +func TestTrimFieldsForDualStackDowngrade(t *testing.T) { + singleStack := api.IPFamilyPolicySingleStack + preferDualStack := api.IPFamilyPolicyPreferDualStack + requireDualStack := api.IPFamilyPolicyRequireDualStack + testCases := []struct { + name string + oldPolicy *api.IPFamilyPolicyType + oldClusterIPs []string + oldFamilies []api.IPFamily + + newPolicy *api.IPFamilyPolicyType + expectedClusterIPs []string + expectedIPFamilies []api.IPFamily + }{ + + { + name: "no change single to single", + oldPolicy: &singleStack, + oldClusterIPs: []string{"10.10.10.10"}, + oldFamilies: []api.IPFamily{api.IPv4Protocol}, + newPolicy: &singleStack, + expectedClusterIPs: []string{"10.10.10.10"}, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + + { + name: "dualstack to dualstack (preferred)", + oldPolicy: &preferDualStack, + oldClusterIPs: []string{"10.10.10.10", "2000::1"}, + oldFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + newPolicy: &preferDualStack, + expectedClusterIPs: []string{"10.10.10.10", "2000::1"}, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + }, + + { + name: "dualstack to dualstack (required)", + oldPolicy: &requireDualStack, + oldClusterIPs: []string{"10.10.10.10", "2000::1"}, + oldFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + newPolicy: &preferDualStack, + expectedClusterIPs: []string{"10.10.10.10", "2000::1"}, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + }, + + { + name: "dualstack (preferred) to single", + oldPolicy: &preferDualStack, + oldClusterIPs: []string{"10.10.10.10", "2000::1"}, + oldFamilies: []api.IPFamily{api.IPv4Protocol, api.IPv6Protocol}, + newPolicy: &singleStack, + expectedClusterIPs: []string{"10.10.10.10"}, + expectedIPFamilies: []api.IPFamily{api.IPv4Protocol}, + }, + + { + name: "dualstack (require) to single", + oldPolicy: &requireDualStack, + oldClusterIPs: []string{"2000::1", "10.10.10.10"}, + oldFamilies: []api.IPFamily{api.IPv6Protocol, api.IPv4Protocol}, + newPolicy: &singleStack, + expectedClusterIPs: []string{"2000::1"}, + expectedIPFamilies: []api.IPFamily{api.IPv6Protocol}, + }, + } + // only when gate is on + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + oldService := &api.Service{ + Spec: api.ServiceSpec{ + IPFamilyPolicy: tc.oldPolicy, + ClusterIPs: tc.oldClusterIPs, + IPFamilies: tc.oldFamilies, + }, + } + + newService := oldService.DeepCopy() + newService.Spec.IPFamilyPolicy = tc.newPolicy + + trimFieldsForDualStackDowngrade(newService, oldService) + + if len(newService.Spec.ClusterIPs) != len(tc.expectedClusterIPs) { + t.Fatalf("unexpected clusterIPs. expected %v and got %v", tc.expectedClusterIPs, newService.Spec.ClusterIPs) + } + + // compare clusterIPS + for i, expectedIP := range tc.expectedClusterIPs { + if expectedIP != newService.Spec.ClusterIPs[i] { + t.Fatalf("unexpected clusterIPs. expected %v and got %v", tc.expectedClusterIPs, newService.Spec.ClusterIPs) + } + } + + // families + if len(newService.Spec.IPFamilies) != len(tc.expectedIPFamilies) { + t.Fatalf("unexpected ipfamilies. expected %v and got %v", tc.expectedIPFamilies, newService.Spec.IPFamilies) + } + + // compare clusterIPS + for i, expectedIPFamily := range tc.expectedIPFamilies { + if expectedIPFamily != newService.Spec.IPFamilies[i] { + t.Fatalf("unexpected ipfamilies. expected %v and got %v", tc.expectedIPFamilies, newService.Spec.IPFamilies) + } + } + + }) + } +} diff --git a/staging/src/k8s.io/api/core/v1/generated.pb.go b/staging/src/k8s.io/api/core/v1/generated.pb.go index 9b29b21e574..ae81d521cef 100644 --- a/staging/src/k8s.io/api/core/v1/generated.pb.go +++ b/staging/src/k8s.io/api/core/v1/generated.pb.go @@ -6087,876 +6087,878 @@ func init() { } var fileDescriptor_83c10c24ec417dc9 = []byte{ - // 13889 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x24, 0xd7, - 0x75, 0x18, 0xac, 0x9e, 0xc1, 0x63, 0xe6, 0xe0, 0x7d, 0xb1, 0xbb, 0xc4, 0x82, 0xbb, 0x8b, 0x65, - 0xaf, 0xb4, 0x5c, 0x8a, 0x24, 0x56, 0x7c, 0x89, 0x34, 0x49, 0xd1, 0x02, 0x30, 0xc0, 0xee, 0x70, - 0x17, 0xd8, 0xe1, 0x1d, 0xec, 0xae, 0x44, 0x53, 0xfa, 0xd4, 0x98, 0xb9, 0x00, 0x9a, 0x98, 0xe9, - 0x1e, 0x76, 0xf7, 0x60, 0x17, 0xfc, 0xe4, 0xfa, 0xfc, 0xc9, 0x4f, 0xf9, 0x91, 0x52, 0xa5, 0x5c, - 0x79, 0xd8, 0x2e, 0x57, 0xca, 0x71, 0xca, 0x56, 0x9c, 0xa4, 0xe2, 0xd8, 0xb1, 0x1d, 0xcb, 0x89, - 0x9d, 0x38, 0x0f, 0x27, 0x3f, 0x1c, 0xc7, 0x95, 0x44, 0xae, 0x72, 0x05, 0xb1, 0xd7, 0xa9, 0xb8, - 0xf4, 0x23, 0xb6, 0x13, 0x3b, 0x3f, 0x82, 0xb8, 0xe2, 0xd4, 0x7d, 0xf6, 0xbd, 0x3d, 0xdd, 0x33, - 0x83, 0x25, 0x00, 0x51, 0x2a, 0xfe, 0x9b, 0xb9, 0xe7, 0xdc, 0x73, 0x6f, 0xdf, 0xe7, 0xb9, 0xe7, - 0x09, 0xaf, 0xec, 0xbc, 0x14, 0xce, 0xbb, 0xfe, 0xd5, 0x9d, 0xf6, 0x06, 0x09, 0x3c, 0x12, 0x91, - 0xf0, 0xea, 0x2e, 0xf1, 0xea, 0x7e, 0x70, 0x55, 0x00, 0x9c, 0x96, 0x7b, 0xb5, 0xe6, 0x07, 0xe4, - 0xea, 0xee, 0x33, 0x57, 0xb7, 0x88, 0x47, 0x02, 0x27, 0x22, 0xf5, 0xf9, 0x56, 0xe0, 0x47, 0x3e, - 0x42, 0x1c, 0x67, 0xde, 0x69, 0xb9, 0xf3, 0x14, 0x67, 0x7e, 0xf7, 0x99, 0xd9, 0xa7, 0xb7, 0xdc, - 0x68, 0xbb, 0xbd, 0x31, 0x5f, 0xf3, 0x9b, 0x57, 0xb7, 0xfc, 0x2d, 0xff, 0x2a, 0x43, 0xdd, 0x68, - 0x6f, 0xb2, 0x7f, 0xec, 0x0f, 0xfb, 0xc5, 0x49, 0xcc, 0x3e, 0x1f, 0x37, 0xd3, 0x74, 0x6a, 0xdb, - 0xae, 0x47, 0x82, 0xbd, 0xab, 0xad, 0x9d, 0x2d, 0xd6, 0x6e, 0x40, 0x42, 0xbf, 0x1d, 0xd4, 0x48, - 0xb2, 0xe1, 0xae, 0xb5, 0xc2, 0xab, 0x4d, 0x12, 0x39, 0x29, 0xdd, 0x9d, 0xbd, 0x9a, 0x55, 0x2b, - 0x68, 0x7b, 0x91, 0xdb, 0xec, 0x6c, 0xe6, 0xe3, 0xbd, 0x2a, 0x84, 0xb5, 0x6d, 0xd2, 0x74, 0x3a, - 0xea, 0x3d, 0x97, 0x55, 0xaf, 0x1d, 0xb9, 0x8d, 0xab, 0xae, 0x17, 0x85, 0x51, 0x90, 0xac, 0x64, - 0x7f, 0xd5, 0x82, 0x8b, 0x0b, 0x77, 0xab, 0xcb, 0x0d, 0x27, 0x8c, 0xdc, 0xda, 0x62, 0xc3, 0xaf, - 0xed, 0x54, 0x23, 0x3f, 0x20, 0x77, 0xfc, 0x46, 0xbb, 0x49, 0xaa, 0x6c, 0x20, 0xd0, 0x53, 0x50, - 0xd8, 0x65, 0xff, 0xcb, 0xa5, 0x19, 0xeb, 0xa2, 0x75, 0xa5, 0xb8, 0x38, 0xf9, 0x1b, 0xfb, 0x73, - 0x1f, 0x7a, 0xb0, 0x3f, 0x57, 0xb8, 0x23, 0xca, 0xb1, 0xc2, 0x40, 0x97, 0x61, 0x68, 0x33, 0x5c, - 0xdf, 0x6b, 0x91, 0x99, 0x1c, 0xc3, 0x1d, 0x17, 0xb8, 0x43, 0x2b, 0x55, 0x5a, 0x8a, 0x05, 0x14, - 0x5d, 0x85, 0x62, 0xcb, 0x09, 0x22, 0x37, 0x72, 0x7d, 0x6f, 0x26, 0x7f, 0xd1, 0xba, 0x32, 0xb8, - 0x38, 0x25, 0x50, 0x8b, 0x15, 0x09, 0xc0, 0x31, 0x0e, 0xed, 0x46, 0x40, 0x9c, 0xfa, 0x2d, 0xaf, - 0xb1, 0x37, 0x33, 0x70, 0xd1, 0xba, 0x52, 0x88, 0xbb, 0x81, 0x45, 0x39, 0x56, 0x18, 0xf6, 0x8f, - 0xe4, 0xa0, 0xb0, 0xb0, 0xb9, 0xe9, 0x7a, 0x6e, 0xb4, 0x87, 0xee, 0xc0, 0xa8, 0xe7, 0xd7, 0x89, - 0xfc, 0xcf, 0xbe, 0x62, 0xe4, 0xd9, 0x8b, 0xf3, 0x9d, 0x4b, 0x69, 0x7e, 0x4d, 0xc3, 0x5b, 0x9c, - 0x7c, 0xb0, 0x3f, 0x37, 0xaa, 0x97, 0x60, 0x83, 0x0e, 0xc2, 0x30, 0xd2, 0xf2, 0xeb, 0x8a, 0x6c, - 0x8e, 0x91, 0x9d, 0x4b, 0x23, 0x5b, 0x89, 0xd1, 0x16, 0x27, 0x1e, 0xec, 0xcf, 0x8d, 0x68, 0x05, - 0x58, 0x27, 0x82, 0x36, 0x60, 0x82, 0xfe, 0xf5, 0x22, 0x57, 0xd1, 0xcd, 0x33, 0xba, 0x97, 0xb2, - 0xe8, 0x6a, 0xa8, 0x8b, 0xd3, 0x0f, 0xf6, 0xe7, 0x26, 0x12, 0x85, 0x38, 0x49, 0xd0, 0x7e, 0x17, - 0xc6, 0x17, 0xa2, 0xc8, 0xa9, 0x6d, 0x93, 0x3a, 0x9f, 0x41, 0xf4, 0x3c, 0x0c, 0x78, 0x4e, 0x93, - 0x88, 0xf9, 0xbd, 0x28, 0x06, 0x76, 0x60, 0xcd, 0x69, 0x92, 0x83, 0xfd, 0xb9, 0xc9, 0xdb, 0x9e, - 0xfb, 0x4e, 0x5b, 0xac, 0x0a, 0x5a, 0x86, 0x19, 0x36, 0x7a, 0x16, 0xa0, 0x4e, 0x76, 0xdd, 0x1a, - 0xa9, 0x38, 0xd1, 0xb6, 0x98, 0x6f, 0x24, 0xea, 0x42, 0x49, 0x41, 0xb0, 0x86, 0x65, 0xdf, 0x87, - 0xe2, 0xc2, 0xae, 0xef, 0xd6, 0x2b, 0x7e, 0x3d, 0x44, 0x3b, 0x30, 0xd1, 0x0a, 0xc8, 0x26, 0x09, - 0x54, 0xd1, 0x8c, 0x75, 0x31, 0x7f, 0x65, 0xe4, 0xd9, 0x2b, 0xa9, 0x1f, 0x6b, 0xa2, 0x2e, 0x7b, - 0x51, 0xb0, 0xb7, 0xf8, 0x88, 0x68, 0x6f, 0x22, 0x01, 0xc5, 0x49, 0xca, 0xf6, 0x3f, 0xcf, 0xc1, - 0xe9, 0x85, 0x77, 0xdb, 0x01, 0x29, 0xb9, 0xe1, 0x4e, 0x72, 0x85, 0xd7, 0xdd, 0x70, 0x67, 0x2d, - 0x1e, 0x01, 0xb5, 0xb4, 0x4a, 0xa2, 0x1c, 0x2b, 0x0c, 0xf4, 0x34, 0x0c, 0xd3, 0xdf, 0xb7, 0x71, - 0x59, 0x7c, 0xf2, 0xb4, 0x40, 0x1e, 0x29, 0x39, 0x91, 0x53, 0xe2, 0x20, 0x2c, 0x71, 0xd0, 0x2a, - 0x8c, 0xd4, 0xd8, 0x86, 0xdc, 0x5a, 0xf5, 0xeb, 0x84, 0x4d, 0x66, 0x71, 0xf1, 0x49, 0x8a, 0xbe, - 0x14, 0x17, 0x1f, 0xec, 0xcf, 0xcd, 0xf0, 0xbe, 0x09, 0x12, 0x1a, 0x0c, 0xeb, 0xf5, 0x91, 0xad, - 0xf6, 0xd7, 0x00, 0xa3, 0x04, 0x29, 0x7b, 0xeb, 0x8a, 0xb6, 0x55, 0x06, 0xd9, 0x56, 0x19, 0x4d, - 0xdf, 0x26, 0xe8, 0x19, 0x18, 0xd8, 0x71, 0xbd, 0xfa, 0xcc, 0x10, 0xa3, 0x75, 0x9e, 0xce, 0xf9, - 0x0d, 0xd7, 0xab, 0x1f, 0xec, 0xcf, 0x4d, 0x19, 0xdd, 0xa1, 0x85, 0x98, 0xa1, 0xda, 0x7f, 0x6a, - 0xc1, 0x1c, 0x83, 0xad, 0xb8, 0x0d, 0x52, 0x21, 0x41, 0xe8, 0x86, 0x11, 0xf1, 0x22, 0x63, 0x40, - 0x9f, 0x05, 0x08, 0x49, 0x2d, 0x20, 0x91, 0x36, 0xa4, 0x6a, 0x61, 0x54, 0x15, 0x04, 0x6b, 0x58, - 0xf4, 0x40, 0x08, 0xb7, 0x9d, 0x80, 0xad, 0x2f, 0x31, 0xb0, 0xea, 0x40, 0xa8, 0x4a, 0x00, 0x8e, - 0x71, 0x8c, 0x03, 0x21, 0xdf, 0xeb, 0x40, 0x40, 0x9f, 0x80, 0x89, 0xb8, 0xb1, 0xb0, 0xe5, 0xd4, - 0xe4, 0x00, 0xb2, 0x2d, 0x53, 0x35, 0x41, 0x38, 0x89, 0x6b, 0xff, 0x6d, 0x4b, 0x2c, 0x1e, 0xfa, - 0xd5, 0xef, 0xf3, 0x6f, 0xb5, 0x7f, 0xc9, 0x82, 0xe1, 0x45, 0xd7, 0xab, 0xbb, 0xde, 0x16, 0xfa, - 0x1c, 0x14, 0xe8, 0xdd, 0x54, 0x77, 0x22, 0x47, 0x9c, 0x7b, 0x1f, 0xd3, 0xf6, 0x96, 0xba, 0x2a, - 0xe6, 0x5b, 0x3b, 0x5b, 0xb4, 0x20, 0x9c, 0xa7, 0xd8, 0x74, 0xb7, 0xdd, 0xda, 0x78, 0x9b, 0xd4, - 0xa2, 0x55, 0x12, 0x39, 0xf1, 0xe7, 0xc4, 0x65, 0x58, 0x51, 0x45, 0x37, 0x60, 0x28, 0x72, 0x82, - 0x2d, 0x12, 0x89, 0x03, 0x30, 0xf5, 0xa0, 0xe2, 0x35, 0x31, 0xdd, 0x91, 0xc4, 0xab, 0x91, 0xf8, - 0x5a, 0x58, 0x67, 0x55, 0xb1, 0x20, 0x61, 0xff, 0xd0, 0x30, 0x9c, 0x5d, 0xaa, 0x96, 0x33, 0xd6, - 0xd5, 0x65, 0x18, 0xaa, 0x07, 0xee, 0x2e, 0x09, 0xc4, 0x38, 0x2b, 0x2a, 0x25, 0x56, 0x8a, 0x05, - 0x14, 0xbd, 0x04, 0xa3, 0xfc, 0x42, 0xba, 0xee, 0x78, 0xf5, 0x86, 0x1c, 0xe2, 0x53, 0x02, 0x7b, - 0xf4, 0x8e, 0x06, 0xc3, 0x06, 0xe6, 0x21, 0x17, 0xd5, 0xe5, 0xc4, 0x66, 0xcc, 0xba, 0xec, 0xbe, - 0x68, 0xc1, 0x24, 0x6f, 0x66, 0x21, 0x8a, 0x02, 0x77, 0xa3, 0x1d, 0x91, 0x70, 0x66, 0x90, 0x9d, - 0x74, 0x4b, 0x69, 0xa3, 0x95, 0x39, 0x02, 0xf3, 0x77, 0x12, 0x54, 0xf8, 0x21, 0x38, 0x23, 0xda, - 0x9d, 0x4c, 0x82, 0x71, 0x47, 0xb3, 0xe8, 0x3b, 0x2d, 0x98, 0xad, 0xf9, 0x5e, 0x14, 0xf8, 0x8d, - 0x06, 0x09, 0x2a, 0xed, 0x8d, 0x86, 0x1b, 0x6e, 0xf3, 0x75, 0x8a, 0xc9, 0x26, 0x3b, 0x09, 0x32, - 0xe6, 0x50, 0x21, 0x89, 0x39, 0xbc, 0xf0, 0x60, 0x7f, 0x6e, 0x76, 0x29, 0x93, 0x14, 0xee, 0xd2, - 0x0c, 0xda, 0x01, 0x44, 0xaf, 0xd2, 0x6a, 0xe4, 0x6c, 0x91, 0xb8, 0xf1, 0xe1, 0xfe, 0x1b, 0x3f, - 0xf3, 0x60, 0x7f, 0x0e, 0xad, 0x75, 0x90, 0xc0, 0x29, 0x64, 0xd1, 0x3b, 0x70, 0x8a, 0x96, 0x76, - 0x7c, 0x6b, 0xa1, 0xff, 0xe6, 0x66, 0x1e, 0xec, 0xcf, 0x9d, 0x5a, 0x4b, 0x21, 0x82, 0x53, 0x49, - 0xa3, 0xef, 0xb0, 0xe0, 0x6c, 0xfc, 0xf9, 0xcb, 0xf7, 0x5b, 0x8e, 0x57, 0x8f, 0x1b, 0x2e, 0xf6, - 0xdf, 0x30, 0x3d, 0x93, 0xcf, 0x2e, 0x65, 0x51, 0xc2, 0xd9, 0x8d, 0xcc, 0x2e, 0xc1, 0xe9, 0xd4, - 0xd5, 0x82, 0x26, 0x21, 0xbf, 0x43, 0x38, 0x17, 0x54, 0xc4, 0xf4, 0x27, 0x3a, 0x05, 0x83, 0xbb, - 0x4e, 0xa3, 0x2d, 0x36, 0x0a, 0xe6, 0x7f, 0x5e, 0xce, 0xbd, 0x64, 0xd9, 0xff, 0x22, 0x0f, 0x13, - 0x4b, 0xd5, 0xf2, 0x43, 0xed, 0x42, 0xfd, 0x1a, 0xca, 0x75, 0xbd, 0x86, 0xe2, 0x4b, 0x2d, 0x9f, - 0x79, 0xa9, 0xfd, 0x7f, 0x29, 0x5b, 0x68, 0x80, 0x6d, 0xa1, 0x6f, 0xc9, 0xd8, 0x42, 0x47, 0xbc, - 0x71, 0x76, 0x33, 0x56, 0xd1, 0x20, 0x9b, 0xcc, 0x54, 0x8e, 0xe5, 0xa6, 0x5f, 0x73, 0x1a, 0xc9, - 0xa3, 0xef, 0x90, 0x4b, 0xe9, 0x68, 0xe6, 0xb1, 0x06, 0xa3, 0x4b, 0x4e, 0xcb, 0xd9, 0x70, 0x1b, - 0x6e, 0xe4, 0x92, 0x10, 0x3d, 0x0e, 0x79, 0xa7, 0x5e, 0x67, 0xdc, 0x56, 0x71, 0xf1, 0xf4, 0x83, - 0xfd, 0xb9, 0xfc, 0x42, 0x9d, 0x5e, 0xfb, 0xa0, 0xb0, 0xf6, 0x30, 0xc5, 0x40, 0x1f, 0x85, 0x81, - 0x7a, 0xe0, 0xb7, 0x66, 0x72, 0x0c, 0x93, 0xee, 0xba, 0x81, 0x52, 0xe0, 0xb7, 0x12, 0xa8, 0x0c, - 0xc7, 0xfe, 0xb5, 0x1c, 0x9c, 0x5b, 0x22, 0xad, 0xed, 0x95, 0x6a, 0xc6, 0xf9, 0x7d, 0x05, 0x0a, - 0x4d, 0xdf, 0x73, 0x23, 0x3f, 0x08, 0x45, 0xd3, 0x6c, 0x45, 0xac, 0x8a, 0x32, 0xac, 0xa0, 0xe8, - 0x22, 0x0c, 0xb4, 0x62, 0xa6, 0x72, 0x54, 0x32, 0xa4, 0x8c, 0x9d, 0x64, 0x10, 0x8a, 0xd1, 0x0e, - 0x49, 0x20, 0x56, 0x8c, 0xc2, 0xb8, 0x1d, 0x92, 0x00, 0x33, 0x48, 0x7c, 0x33, 0xd3, 0x3b, 0x5b, - 0x9c, 0xd0, 0x89, 0x9b, 0x99, 0x42, 0xb0, 0x86, 0x85, 0x2a, 0x50, 0x0c, 0x13, 0x33, 0xdb, 0xd7, - 0x36, 0x1d, 0x63, 0x57, 0xb7, 0x9a, 0xc9, 0x98, 0x88, 0x71, 0xa3, 0x0c, 0xf5, 0xbc, 0xba, 0xbf, - 0x92, 0x03, 0xc4, 0x87, 0xf0, 0x1b, 0x6c, 0xe0, 0x6e, 0x77, 0x0e, 0x5c, 0xff, 0x5b, 0xe2, 0xa8, - 0x46, 0xef, 0xcf, 0x2c, 0x38, 0xb7, 0xe4, 0x7a, 0x75, 0x12, 0x64, 0x2c, 0xc0, 0xe3, 0x79, 0xcb, - 0x1e, 0x8e, 0x69, 0x30, 0x96, 0xd8, 0xc0, 0x11, 0x2c, 0x31, 0xfb, 0x8f, 0x2d, 0x40, 0xfc, 0xb3, - 0xdf, 0x77, 0x1f, 0x7b, 0xbb, 0xf3, 0x63, 0x8f, 0x60, 0x59, 0xd8, 0x37, 0x61, 0x7c, 0xa9, 0xe1, - 0x12, 0x2f, 0x2a, 0x57, 0x96, 0x7c, 0x6f, 0xd3, 0xdd, 0x42, 0x2f, 0xc3, 0x78, 0xe4, 0x36, 0x89, - 0xdf, 0x8e, 0xaa, 0xa4, 0xe6, 0x7b, 0xec, 0x25, 0x69, 0x5d, 0x19, 0x5c, 0x44, 0x0f, 0xf6, 0xe7, - 0xc6, 0xd7, 0x0d, 0x08, 0x4e, 0x60, 0xda, 0xbf, 0x4b, 0xc7, 0xcf, 0x6f, 0xb6, 0x7c, 0x8f, 0x78, - 0xd1, 0x92, 0xef, 0xd5, 0xb9, 0xc4, 0xe1, 0x65, 0x18, 0x88, 0xe8, 0x78, 0xf0, 0xb1, 0xbb, 0x2c, - 0x37, 0x0a, 0x1d, 0x85, 0x83, 0xfd, 0xb9, 0x33, 0x9d, 0x35, 0xd8, 0x38, 0xb1, 0x3a, 0xe8, 0x5b, - 0x60, 0x28, 0x8c, 0x9c, 0xa8, 0x1d, 0x8a, 0xd1, 0x7c, 0x4c, 0x8e, 0x66, 0x95, 0x95, 0x1e, 0xec, - 0xcf, 0x4d, 0xa8, 0x6a, 0xbc, 0x08, 0x8b, 0x0a, 0xe8, 0x09, 0x18, 0x6e, 0x92, 0x30, 0x74, 0xb6, - 0xe4, 0x6d, 0x38, 0x21, 0xea, 0x0e, 0xaf, 0xf2, 0x62, 0x2c, 0xe1, 0xe8, 0x12, 0x0c, 0x92, 0x20, - 0xf0, 0x03, 0xb1, 0x47, 0xc7, 0x04, 0xe2, 0xe0, 0x32, 0x2d, 0xc4, 0x1c, 0x66, 0xff, 0x5b, 0x0b, - 0x26, 0x54, 0x5f, 0x79, 0x5b, 0x27, 0xf0, 0x2a, 0x78, 0x13, 0xa0, 0x26, 0x3f, 0x30, 0x64, 0xb7, - 0xc7, 0xc8, 0xb3, 0x97, 0x53, 0x2f, 0xea, 0x8e, 0x61, 0x8c, 0x29, 0xab, 0xa2, 0x10, 0x6b, 0xd4, - 0xec, 0x7f, 0x6c, 0xc1, 0x74, 0xe2, 0x8b, 0x6e, 0xba, 0x61, 0x84, 0xde, 0xea, 0xf8, 0xaa, 0xf9, - 0xfe, 0xbe, 0x8a, 0xd6, 0x66, 0xdf, 0xa4, 0x96, 0xb2, 0x2c, 0xd1, 0xbe, 0xe8, 0x3a, 0x0c, 0xba, - 0x11, 0x69, 0xca, 0x8f, 0xb9, 0xd4, 0xf5, 0x63, 0x78, 0xaf, 0xe2, 0x19, 0x29, 0xd3, 0x9a, 0x98, - 0x13, 0xb0, 0x7f, 0x2d, 0x0f, 0x45, 0xbe, 0x6c, 0x57, 0x9d, 0xd6, 0x09, 0xcc, 0xc5, 0x93, 0x50, - 0x74, 0x9b, 0xcd, 0x76, 0xe4, 0x6c, 0x88, 0xe3, 0xbc, 0xc0, 0xb7, 0x56, 0x59, 0x16, 0xe2, 0x18, - 0x8e, 0xca, 0x30, 0xc0, 0xba, 0xc2, 0xbf, 0xf2, 0xf1, 0xf4, 0xaf, 0x14, 0x7d, 0x9f, 0x2f, 0x39, - 0x91, 0xc3, 0x39, 0x29, 0x75, 0x8f, 0xd0, 0x22, 0xcc, 0x48, 0x20, 0x07, 0x60, 0xc3, 0xf5, 0x9c, - 0x60, 0x8f, 0x96, 0xcd, 0xe4, 0x19, 0xc1, 0xa7, 0xbb, 0x13, 0x5c, 0x54, 0xf8, 0x9c, 0xac, 0xfa, - 0xb0, 0x18, 0x80, 0x35, 0xa2, 0xb3, 0x2f, 0x42, 0x51, 0x21, 0x1f, 0x86, 0x21, 0x9a, 0xfd, 0x04, - 0x4c, 0x24, 0xda, 0xea, 0x55, 0x7d, 0x54, 0xe7, 0xa7, 0x7e, 0x99, 0x1d, 0x19, 0xa2, 0xd7, 0xcb, - 0xde, 0xae, 0x38, 0x72, 0xdf, 0x85, 0x53, 0x8d, 0x94, 0x93, 0x4c, 0xcc, 0x6b, 0xff, 0x27, 0xdf, - 0x39, 0xf1, 0xd9, 0xa7, 0xd2, 0xa0, 0x38, 0xb5, 0x0d, 0xca, 0x23, 0xf8, 0x2d, 0xba, 0x41, 0x9c, - 0x86, 0xce, 0x6e, 0xdf, 0x12, 0x65, 0x58, 0x41, 0xe9, 0x79, 0x77, 0x4a, 0x75, 0xfe, 0x06, 0xd9, - 0xab, 0x92, 0x06, 0xa9, 0x45, 0x7e, 0xf0, 0x75, 0xed, 0xfe, 0x79, 0x3e, 0xfa, 0xfc, 0xb8, 0x1c, - 0x11, 0x04, 0xf2, 0x37, 0xc8, 0x1e, 0x9f, 0x0a, 0xfd, 0xeb, 0xf2, 0x5d, 0xbf, 0xee, 0x67, 0x2d, - 0x18, 0x53, 0x5f, 0x77, 0x02, 0xe7, 0xc2, 0xa2, 0x79, 0x2e, 0x9c, 0xef, 0xba, 0xc0, 0x33, 0x4e, - 0x84, 0xaf, 0xe4, 0xe0, 0xac, 0xc2, 0xa1, 0x6f, 0x03, 0xfe, 0x47, 0xac, 0xaa, 0xab, 0x50, 0xf4, - 0x94, 0xd4, 0xca, 0x32, 0xc5, 0x45, 0xb1, 0xcc, 0x2a, 0xc6, 0xa1, 0x2c, 0x9e, 0x17, 0x8b, 0x96, - 0x46, 0x75, 0x71, 0xae, 0x10, 0xdd, 0x2e, 0x42, 0xbe, 0xed, 0xd6, 0xc5, 0x05, 0xf3, 0x31, 0x39, - 0xda, 0xb7, 0xcb, 0xa5, 0x83, 0xfd, 0xb9, 0xc7, 0xb2, 0x54, 0x09, 0xf4, 0x66, 0x0b, 0xe7, 0x6f, - 0x97, 0x4b, 0x98, 0x56, 0x46, 0x0b, 0x30, 0x21, 0xb5, 0x25, 0x77, 0x28, 0xbb, 0xe5, 0x7b, 0xe2, - 0x1e, 0x52, 0x32, 0x59, 0x6c, 0x82, 0x71, 0x12, 0x1f, 0x95, 0x60, 0x72, 0xa7, 0xbd, 0x41, 0x1a, - 0x24, 0xe2, 0x1f, 0x7c, 0x83, 0x70, 0x89, 0x65, 0x31, 0x7e, 0x99, 0xdd, 0x48, 0xc0, 0x71, 0x47, - 0x0d, 0xfb, 0x2f, 0xd8, 0x7d, 0x20, 0x46, 0xaf, 0x12, 0xf8, 0x74, 0x61, 0x51, 0xea, 0x5f, 0xcf, - 0xe5, 0xdc, 0xcf, 0xaa, 0xb8, 0x41, 0xf6, 0xd6, 0x7d, 0xca, 0x99, 0xa7, 0xaf, 0x0a, 0x63, 0xcd, - 0x0f, 0x74, 0x5d, 0xf3, 0x3f, 0x9f, 0x83, 0xd3, 0x6a, 0x04, 0x0c, 0x26, 0xf0, 0x1b, 0x7d, 0x0c, - 0x9e, 0x81, 0x91, 0x3a, 0xd9, 0x74, 0xda, 0x8d, 0x48, 0x89, 0xcf, 0x07, 0xb9, 0x0a, 0xa5, 0x14, - 0x17, 0x63, 0x1d, 0xe7, 0x10, 0xc3, 0xf6, 0x3f, 0x47, 0xd8, 0x45, 0x1c, 0x39, 0x74, 0x8d, 0xab, - 0x5d, 0x63, 0x65, 0xee, 0x9a, 0x4b, 0x30, 0xe8, 0x36, 0x29, 0x63, 0x96, 0x33, 0xf9, 0xad, 0x32, - 0x2d, 0xc4, 0x1c, 0x86, 0x3e, 0x02, 0xc3, 0x35, 0xbf, 0xd9, 0x74, 0xbc, 0x3a, 0xbb, 0xf2, 0x8a, - 0x8b, 0x23, 0x94, 0x77, 0x5b, 0xe2, 0x45, 0x58, 0xc2, 0xd0, 0x39, 0x18, 0x70, 0x82, 0x2d, 0x2e, - 0xc3, 0x28, 0x2e, 0x16, 0x68, 0x4b, 0x0b, 0xc1, 0x56, 0x88, 0x59, 0x29, 0x7d, 0x82, 0xdd, 0xf3, - 0x83, 0x1d, 0xd7, 0xdb, 0x2a, 0xb9, 0x81, 0xd8, 0x12, 0xea, 0x2e, 0xbc, 0xab, 0x20, 0x58, 0xc3, - 0x42, 0x2b, 0x30, 0xd8, 0xf2, 0x83, 0x28, 0x9c, 0x19, 0x62, 0xc3, 0xfd, 0x58, 0xc6, 0x41, 0xc4, - 0xbf, 0xb6, 0xe2, 0x07, 0x51, 0xfc, 0x01, 0xf4, 0x5f, 0x88, 0x79, 0x75, 0x74, 0x13, 0x86, 0x89, - 0xb7, 0xbb, 0x12, 0xf8, 0xcd, 0x99, 0xe9, 0x6c, 0x4a, 0xcb, 0x1c, 0x85, 0x2f, 0xb3, 0x98, 0x47, - 0x15, 0xc5, 0x58, 0x92, 0x40, 0xdf, 0x02, 0x79, 0xe2, 0xed, 0xce, 0x0c, 0x33, 0x4a, 0xb3, 0x19, - 0x94, 0xee, 0x38, 0x41, 0x7c, 0xe6, 0x2f, 0x7b, 0xbb, 0x98, 0xd6, 0x41, 0x9f, 0x86, 0xa2, 0x3c, - 0x30, 0x42, 0x21, 0xac, 0x4b, 0x5d, 0xb0, 0xf2, 0x98, 0xc1, 0xe4, 0x9d, 0xb6, 0x1b, 0x90, 0x26, - 0xf1, 0xa2, 0x30, 0x3e, 0x21, 0x25, 0x34, 0xc4, 0x31, 0x35, 0xf4, 0x69, 0x29, 0x21, 0x5e, 0xf5, - 0xdb, 0x5e, 0x14, 0xce, 0x14, 0x59, 0xf7, 0x52, 0x75, 0x77, 0x77, 0x62, 0xbc, 0xa4, 0x08, 0x99, - 0x57, 0xc6, 0x06, 0x29, 0xf4, 0x19, 0x18, 0xe3, 0xff, 0xb9, 0x06, 0x2c, 0x9c, 0x39, 0xcd, 0x68, - 0x5f, 0xcc, 0xa6, 0xcd, 0x11, 0x17, 0x4f, 0x0b, 0xe2, 0x63, 0x7a, 0x69, 0x88, 0x4d, 0x6a, 0x08, - 0xc3, 0x58, 0xc3, 0xdd, 0x25, 0x1e, 0x09, 0xc3, 0x4a, 0xe0, 0x6f, 0x90, 0x19, 0x60, 0x03, 0x73, - 0x36, 0x5d, 0x63, 0xe6, 0x6f, 0x90, 0xc5, 0x29, 0x4a, 0xf3, 0xa6, 0x5e, 0x07, 0x9b, 0x24, 0xd0, - 0x6d, 0x18, 0xa7, 0x2f, 0x36, 0x37, 0x26, 0x3a, 0xd2, 0x8b, 0x28, 0x7b, 0x57, 0x61, 0xa3, 0x12, - 0x4e, 0x10, 0x41, 0xb7, 0x60, 0x34, 0x8c, 0x9c, 0x20, 0x6a, 0xb7, 0x38, 0xd1, 0x33, 0xbd, 0x88, - 0x32, 0x85, 0x6b, 0x55, 0xab, 0x82, 0x0d, 0x02, 0xe8, 0x75, 0x28, 0x36, 0xdc, 0x4d, 0x52, 0xdb, - 0xab, 0x35, 0xc8, 0xcc, 0x28, 0xa3, 0x96, 0x7a, 0xa8, 0xdc, 0x94, 0x48, 0x9c, 0xcf, 0x55, 0x7f, - 0x71, 0x5c, 0x1d, 0xdd, 0x81, 0x33, 0x11, 0x09, 0x9a, 0xae, 0xe7, 0xd0, 0xc3, 0x40, 0x3c, 0xad, - 0x98, 0x22, 0x73, 0x8c, 0xed, 0xb6, 0x0b, 0x62, 0x36, 0xce, 0xac, 0xa7, 0x62, 0xe1, 0x8c, 0xda, - 0xe8, 0x3e, 0xcc, 0xa4, 0x40, 0xfc, 0x86, 0x5b, 0xdb, 0x9b, 0x39, 0xc5, 0x28, 0xbf, 0x2a, 0x28, - 0xcf, 0xac, 0x67, 0xe0, 0x1d, 0x74, 0x81, 0xe1, 0x4c, 0xea, 0xe8, 0x16, 0x4c, 0xb0, 0x13, 0xa8, - 0xd2, 0x6e, 0x34, 0x44, 0x83, 0xe3, 0xac, 0xc1, 0x8f, 0xc8, 0xfb, 0xb8, 0x6c, 0x82, 0x0f, 0xf6, - 0xe7, 0x20, 0xfe, 0x87, 0x93, 0xb5, 0xd1, 0x06, 0xd3, 0x99, 0xb5, 0x03, 0x37, 0xda, 0xa3, 0xe7, - 0x06, 0xb9, 0x1f, 0xcd, 0x4c, 0x74, 0x95, 0x57, 0xe8, 0xa8, 0x4a, 0xb1, 0xa6, 0x17, 0xe2, 0x24, - 0x41, 0x7a, 0xa4, 0x86, 0x51, 0xdd, 0xf5, 0x66, 0x26, 0xf9, 0xbb, 0x44, 0x9e, 0x48, 0x55, 0x5a, - 0x88, 0x39, 0x8c, 0xe9, 0xcb, 0xe8, 0x8f, 0x5b, 0xf4, 0xe6, 0x9a, 0x62, 0x88, 0xb1, 0xbe, 0x4c, - 0x02, 0x70, 0x8c, 0x43, 0x99, 0xc9, 0x28, 0xda, 0x9b, 0x41, 0x0c, 0x55, 0x1d, 0x2c, 0xeb, 0xeb, - 0x9f, 0xc6, 0xb4, 0xdc, 0xde, 0x80, 0x71, 0x75, 0x10, 0xb2, 0x31, 0x41, 0x73, 0x30, 0xc8, 0xd8, - 0x27, 0x21, 0x5d, 0x2b, 0xd2, 0x2e, 0x30, 0xd6, 0x0a, 0xf3, 0x72, 0xd6, 0x05, 0xf7, 0x5d, 0xb2, - 0xb8, 0x17, 0x11, 0xfe, 0xa6, 0xcf, 0x6b, 0x5d, 0x90, 0x00, 0x1c, 0xe3, 0xd8, 0xff, 0x87, 0xb3, - 0xa1, 0xf1, 0x69, 0xdb, 0xc7, 0xfd, 0xf2, 0x14, 0x14, 0xb6, 0xfd, 0x30, 0xa2, 0xd8, 0xac, 0x8d, - 0xc1, 0x98, 0xf1, 0xbc, 0x2e, 0xca, 0xb1, 0xc2, 0x40, 0xaf, 0xc0, 0x58, 0x4d, 0x6f, 0x40, 0x5c, - 0x8e, 0xea, 0x18, 0x31, 0x5a, 0xc7, 0x26, 0x2e, 0x7a, 0x09, 0x0a, 0xcc, 0x06, 0xa4, 0xe6, 0x37, - 0x04, 0xd7, 0x26, 0x6f, 0xf8, 0x42, 0x45, 0x94, 0x1f, 0x68, 0xbf, 0xb1, 0xc2, 0x46, 0x97, 0x61, - 0x88, 0x76, 0xa1, 0x5c, 0x11, 0xd7, 0x92, 0x12, 0x14, 0x5d, 0x67, 0xa5, 0x58, 0x40, 0xed, 0xbf, - 0x9c, 0xd3, 0x46, 0x99, 0xbe, 0x87, 0x09, 0xaa, 0xc0, 0xf0, 0x3d, 0xc7, 0x8d, 0x5c, 0x6f, 0x4b, - 0xf0, 0x1f, 0x4f, 0x74, 0xbd, 0xa3, 0x58, 0xa5, 0xbb, 0xbc, 0x02, 0xbf, 0x45, 0xc5, 0x1f, 0x2c, - 0xc9, 0x50, 0x8a, 0x41, 0xdb, 0xf3, 0x28, 0xc5, 0x5c, 0xbf, 0x14, 0x31, 0xaf, 0xc0, 0x29, 0x8a, - 0x3f, 0x58, 0x92, 0x41, 0x6f, 0x01, 0xc8, 0x1d, 0x46, 0xea, 0xc2, 0xf6, 0xe2, 0xa9, 0xde, 0x44, - 0xd7, 0x55, 0x9d, 0xc5, 0x71, 0x7a, 0x47, 0xc7, 0xff, 0xb1, 0x46, 0xcf, 0x8e, 0x18, 0x9f, 0xd6, - 0xd9, 0x19, 0xf4, 0x6d, 0x74, 0x89, 0x3b, 0x41, 0x44, 0xea, 0x0b, 0x91, 0x18, 0x9c, 0x8f, 0xf6, - 0xf7, 0x48, 0x59, 0x77, 0x9b, 0x44, 0xdf, 0x0e, 0x82, 0x08, 0x8e, 0xe9, 0xd9, 0xbf, 0x98, 0x87, - 0x99, 0xac, 0xee, 0xd2, 0x45, 0x47, 0xee, 0xbb, 0xd1, 0x12, 0x65, 0xaf, 0x2c, 0x73, 0xd1, 0x2d, - 0x8b, 0x72, 0xac, 0x30, 0xe8, 0xec, 0x87, 0xee, 0x96, 0x7c, 0x63, 0x0e, 0xc6, 0xb3, 0x5f, 0x65, - 0xa5, 0x58, 0x40, 0x29, 0x5e, 0x40, 0x9c, 0x50, 0x18, 0xf7, 0x68, 0xab, 0x04, 0xb3, 0x52, 0x2c, - 0xa0, 0xba, 0xb4, 0x6b, 0xa0, 0x87, 0xb4, 0xcb, 0x18, 0xa2, 0xc1, 0xa3, 0x1d, 0x22, 0xf4, 0x59, - 0x80, 0x4d, 0xd7, 0x73, 0xc3, 0x6d, 0x46, 0x7d, 0xe8, 0xd0, 0xd4, 0x15, 0x73, 0xb6, 0xa2, 0xa8, - 0x60, 0x8d, 0x22, 0x7a, 0x01, 0x46, 0xd4, 0x06, 0x2c, 0x97, 0x98, 0xa6, 0x53, 0xb3, 0x1c, 0x89, - 0x4f, 0xa3, 0x12, 0xd6, 0xf1, 0xec, 0xb7, 0x93, 0xeb, 0x45, 0xec, 0x00, 0x6d, 0x7c, 0xad, 0x7e, - 0xc7, 0x37, 0xd7, 0x7d, 0x7c, 0xed, 0xaf, 0xe5, 0x61, 0xc2, 0x68, 0xac, 0x1d, 0xf6, 0x71, 0x66, + // 13928 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x6b, 0x70, 0x1c, 0xd9, + 0x79, 0x18, 0xaa, 0x9e, 0xc1, 0x6b, 0x3e, 0xbc, 0x0f, 0x48, 0x2e, 0x88, 0x25, 0x09, 0x6e, 0x53, + 0xe2, 0x72, 0xb5, 0xbb, 0xa0, 0xf6, 0x25, 0xad, 0x77, 0xa5, 0xb5, 0x00, 0x0c, 0x40, 0xce, 0x92, + 0x00, 0x67, 0xcf, 0x80, 0xa4, 0x24, 0xaf, 0x74, 0xd5, 0x98, 0x39, 0x00, 0x5a, 0x98, 0xe9, 0x9e, + 0xed, 0xee, 0x01, 0x89, 0xbd, 0x72, 0x5d, 0x5f, 0xf9, 0x29, 0x3f, 0x6e, 0xa9, 0x6e, 0xb9, 0xf2, + 0xb0, 0x5d, 0xae, 0x94, 0xe3, 0x94, 0xad, 0x38, 0x49, 0xc5, 0xb1, 0x63, 0x3b, 0x96, 0x13, 0x3b, + 0x76, 0x1e, 0x4e, 0x7e, 0x38, 0x8e, 0x2b, 0x89, 0x5c, 0xe5, 0x0a, 0x62, 0xd3, 0xa9, 0xb8, 0xf4, + 0x23, 0xb6, 0x13, 0x3b, 0x3f, 0x82, 0xb8, 0xe2, 0xd4, 0x79, 0xf6, 0x39, 0xfd, 0x98, 0x19, 0x70, + 0x41, 0x68, 0xa5, 0xda, 0x7f, 0x33, 0xe7, 0xfb, 0xce, 0x77, 0x4e, 0x9f, 0xe7, 0x77, 0xbe, 0x27, + 0xbc, 0xba, 0xfb, 0x72, 0xb8, 0xe0, 0xfa, 0x57, 0x77, 0x3b, 0x9b, 0x24, 0xf0, 0x48, 0x44, 0xc2, + 0xab, 0x7b, 0xc4, 0x6b, 0xf8, 0xc1, 0x55, 0x01, 0x70, 0xda, 0xee, 0xd5, 0xba, 0x1f, 0x90, 0xab, + 0x7b, 0xcf, 0x5d, 0xdd, 0x26, 0x1e, 0x09, 0x9c, 0x88, 0x34, 0x16, 0xda, 0x81, 0x1f, 0xf9, 0x08, + 0x71, 0x9c, 0x05, 0xa7, 0xed, 0x2e, 0x50, 0x9c, 0x85, 0xbd, 0xe7, 0xe6, 0x9e, 0xdd, 0x76, 0xa3, + 0x9d, 0xce, 0xe6, 0x42, 0xdd, 0x6f, 0x5d, 0xdd, 0xf6, 0xb7, 0xfd, 0xab, 0x0c, 0x75, 0xb3, 0xb3, + 0xc5, 0xfe, 0xb1, 0x3f, 0xec, 0x17, 0x27, 0x31, 0xf7, 0x62, 0xdc, 0x4c, 0xcb, 0xa9, 0xef, 0xb8, + 0x1e, 0x09, 0xf6, 0xaf, 0xb6, 0x77, 0xb7, 0x59, 0xbb, 0x01, 0x09, 0xfd, 0x4e, 0x50, 0x27, 0xc9, + 0x86, 0xbb, 0xd6, 0x0a, 0xaf, 0xb6, 0x48, 0xe4, 0x64, 0x74, 0x77, 0xee, 0x6a, 0x5e, 0xad, 0xa0, + 0xe3, 0x45, 0x6e, 0x2b, 0xdd, 0xcc, 0x87, 0x7b, 0x55, 0x08, 0xeb, 0x3b, 0xa4, 0xe5, 0xa4, 0xea, + 0xbd, 0x90, 0x57, 0xaf, 0x13, 0xb9, 0xcd, 0xab, 0xae, 0x17, 0x85, 0x51, 0x90, 0xac, 0x64, 0x7f, + 0xd5, 0x82, 0x8b, 0x8b, 0x77, 0x6b, 0x2b, 0x4d, 0x27, 0x8c, 0xdc, 0xfa, 0x52, 0xd3, 0xaf, 0xef, + 0xd6, 0x22, 0x3f, 0x20, 0x77, 0xfc, 0x66, 0xa7, 0x45, 0x6a, 0x6c, 0x20, 0xd0, 0x33, 0x30, 0xb2, + 0xc7, 0xfe, 0x57, 0xca, 0xb3, 0xd6, 0x45, 0xeb, 0x4a, 0x69, 0x69, 0xea, 0x37, 0x0f, 0xe6, 0xdf, + 0xf7, 0xe0, 0x60, 0x7e, 0xe4, 0x8e, 0x28, 0xc7, 0x0a, 0x03, 0x5d, 0x86, 0xa1, 0xad, 0x70, 0x63, + 0xbf, 0x4d, 0x66, 0x0b, 0x0c, 0x77, 0x42, 0xe0, 0x0e, 0xad, 0xd6, 0x68, 0x29, 0x16, 0x50, 0x74, + 0x15, 0x4a, 0x6d, 0x27, 0x88, 0xdc, 0xc8, 0xf5, 0xbd, 0xd9, 0xe2, 0x45, 0xeb, 0xca, 0xe0, 0xd2, + 0xb4, 0x40, 0x2d, 0x55, 0x25, 0x00, 0xc7, 0x38, 0xb4, 0x1b, 0x01, 0x71, 0x1a, 0xb7, 0xbc, 0xe6, + 0xfe, 0xec, 0xc0, 0x45, 0xeb, 0xca, 0x48, 0xdc, 0x0d, 0x2c, 0xca, 0xb1, 0xc2, 0xb0, 0x7f, 0xa4, + 0x00, 0x23, 0x8b, 0x5b, 0x5b, 0xae, 0xe7, 0x46, 0xfb, 0xe8, 0x0e, 0x8c, 0x79, 0x7e, 0x83, 0xc8, + 0xff, 0xec, 0x2b, 0x46, 0x9f, 0xbf, 0xb8, 0x90, 0x5e, 0x4a, 0x0b, 0xeb, 0x1a, 0xde, 0xd2, 0xd4, + 0x83, 0x83, 0xf9, 0x31, 0xbd, 0x04, 0x1b, 0x74, 0x10, 0x86, 0xd1, 0xb6, 0xdf, 0x50, 0x64, 0x0b, + 0x8c, 0xec, 0x7c, 0x16, 0xd9, 0x6a, 0x8c, 0xb6, 0x34, 0xf9, 0xe0, 0x60, 0x7e, 0x54, 0x2b, 0xc0, + 0x3a, 0x11, 0xb4, 0x09, 0x93, 0xf4, 0xaf, 0x17, 0xb9, 0x8a, 0x6e, 0x91, 0xd1, 0xbd, 0x94, 0x47, + 0x57, 0x43, 0x5d, 0x9a, 0x79, 0x70, 0x30, 0x3f, 0x99, 0x28, 0xc4, 0x49, 0x82, 0xf6, 0xdb, 0x30, + 0xb1, 0x18, 0x45, 0x4e, 0x7d, 0x87, 0x34, 0xf8, 0x0c, 0xa2, 0x17, 0x61, 0xc0, 0x73, 0x5a, 0x44, + 0xcc, 0xef, 0x45, 0x31, 0xb0, 0x03, 0xeb, 0x4e, 0x8b, 0x1c, 0x1e, 0xcc, 0x4f, 0xdd, 0xf6, 0xdc, + 0xb7, 0x3a, 0x62, 0x55, 0xd0, 0x32, 0xcc, 0xb0, 0xd1, 0xf3, 0x00, 0x0d, 0xb2, 0xe7, 0xd6, 0x49, + 0xd5, 0x89, 0x76, 0xc4, 0x7c, 0x23, 0x51, 0x17, 0xca, 0x0a, 0x82, 0x35, 0x2c, 0xfb, 0x3e, 0x94, + 0x16, 0xf7, 0x7c, 0xb7, 0x51, 0xf5, 0x1b, 0x21, 0xda, 0x85, 0xc9, 0x76, 0x40, 0xb6, 0x48, 0xa0, + 0x8a, 0x66, 0xad, 0x8b, 0xc5, 0x2b, 0xa3, 0xcf, 0x5f, 0xc9, 0xfc, 0x58, 0x13, 0x75, 0xc5, 0x8b, + 0x82, 0xfd, 0xa5, 0xc7, 0x44, 0x7b, 0x93, 0x09, 0x28, 0x4e, 0x52, 0xb6, 0xff, 0x59, 0x01, 0x4e, + 0x2f, 0xbe, 0xdd, 0x09, 0x48, 0xd9, 0x0d, 0x77, 0x93, 0x2b, 0xbc, 0xe1, 0x86, 0xbb, 0xeb, 0xf1, + 0x08, 0xa8, 0xa5, 0x55, 0x16, 0xe5, 0x58, 0x61, 0xa0, 0x67, 0x61, 0x98, 0xfe, 0xbe, 0x8d, 0x2b, + 0xe2, 0x93, 0x67, 0x04, 0xf2, 0x68, 0xd9, 0x89, 0x9c, 0x32, 0x07, 0x61, 0x89, 0x83, 0xd6, 0x60, + 0xb4, 0xce, 0x36, 0xe4, 0xf6, 0x9a, 0xdf, 0x20, 0x6c, 0x32, 0x4b, 0x4b, 0x4f, 0x53, 0xf4, 0xe5, + 0xb8, 0xf8, 0xf0, 0x60, 0x7e, 0x96, 0xf7, 0x4d, 0x90, 0xd0, 0x60, 0x58, 0xaf, 0x8f, 0x6c, 0xb5, + 0xbf, 0x06, 0x18, 0x25, 0xc8, 0xd8, 0x5b, 0x57, 0xb4, 0xad, 0x32, 0xc8, 0xb6, 0xca, 0x58, 0xf6, + 0x36, 0x41, 0xcf, 0xc1, 0xc0, 0xae, 0xeb, 0x35, 0x66, 0x87, 0x18, 0xad, 0xf3, 0x74, 0xce, 0x6f, + 0xb8, 0x5e, 0xe3, 0xf0, 0x60, 0x7e, 0xda, 0xe8, 0x0e, 0x2d, 0xc4, 0x0c, 0xd5, 0xfe, 0x33, 0x0b, + 0xe6, 0x19, 0x6c, 0xd5, 0x6d, 0x92, 0x2a, 0x09, 0x42, 0x37, 0x8c, 0x88, 0x17, 0x19, 0x03, 0xfa, + 0x3c, 0x40, 0x48, 0xea, 0x01, 0x89, 0xb4, 0x21, 0x55, 0x0b, 0xa3, 0xa6, 0x20, 0x58, 0xc3, 0xa2, + 0x07, 0x42, 0xb8, 0xe3, 0x04, 0x6c, 0x7d, 0x89, 0x81, 0x55, 0x07, 0x42, 0x4d, 0x02, 0x70, 0x8c, + 0x63, 0x1c, 0x08, 0xc5, 0x5e, 0x07, 0x02, 0xfa, 0x18, 0x4c, 0xc6, 0x8d, 0x85, 0x6d, 0xa7, 0x2e, + 0x07, 0x90, 0x6d, 0x99, 0x9a, 0x09, 0xc2, 0x49, 0x5c, 0xfb, 0x6f, 0x5b, 0x62, 0xf1, 0xd0, 0xaf, + 0x7e, 0x97, 0x7f, 0xab, 0xfd, 0x4b, 0x16, 0x0c, 0x2f, 0xb9, 0x5e, 0xc3, 0xf5, 0xb6, 0xd1, 0x67, + 0x61, 0x84, 0xde, 0x4d, 0x0d, 0x27, 0x72, 0xc4, 0xb9, 0xf7, 0x21, 0x6d, 0x6f, 0xa9, 0xab, 0x62, + 0xa1, 0xbd, 0xbb, 0x4d, 0x0b, 0xc2, 0x05, 0x8a, 0x4d, 0x77, 0xdb, 0xad, 0xcd, 0xcf, 0x91, 0x7a, + 0xb4, 0x46, 0x22, 0x27, 0xfe, 0x9c, 0xb8, 0x0c, 0x2b, 0xaa, 0xe8, 0x06, 0x0c, 0x45, 0x4e, 0xb0, + 0x4d, 0x22, 0x71, 0x00, 0x66, 0x1e, 0x54, 0xbc, 0x26, 0xa6, 0x3b, 0x92, 0x78, 0x75, 0x12, 0x5f, + 0x0b, 0x1b, 0xac, 0x2a, 0x16, 0x24, 0xec, 0x1f, 0x1a, 0x86, 0xb3, 0xcb, 0xb5, 0x4a, 0xce, 0xba, + 0xba, 0x0c, 0x43, 0x8d, 0xc0, 0xdd, 0x23, 0x81, 0x18, 0x67, 0x45, 0xa5, 0xcc, 0x4a, 0xb1, 0x80, + 0xa2, 0x97, 0x61, 0x8c, 0x5f, 0x48, 0xd7, 0x1d, 0xaf, 0xd1, 0x94, 0x43, 0x7c, 0x4a, 0x60, 0x8f, + 0xdd, 0xd1, 0x60, 0xd8, 0xc0, 0x3c, 0xe2, 0xa2, 0xba, 0x9c, 0xd8, 0x8c, 0x79, 0x97, 0xdd, 0x17, + 0x2d, 0x98, 0xe2, 0xcd, 0x2c, 0x46, 0x51, 0xe0, 0x6e, 0x76, 0x22, 0x12, 0xce, 0x0e, 0xb2, 0x93, + 0x6e, 0x39, 0x6b, 0xb4, 0x72, 0x47, 0x60, 0xe1, 0x4e, 0x82, 0x0a, 0x3f, 0x04, 0x67, 0x45, 0xbb, + 0x53, 0x49, 0x30, 0x4e, 0x35, 0x8b, 0xbe, 0xd3, 0x82, 0xb9, 0xba, 0xef, 0x45, 0x81, 0xdf, 0x6c, + 0x92, 0xa0, 0xda, 0xd9, 0x6c, 0xba, 0xe1, 0x0e, 0x5f, 0xa7, 0x98, 0x6c, 0xb1, 0x93, 0x20, 0x67, + 0x0e, 0x15, 0x92, 0x98, 0xc3, 0x0b, 0x0f, 0x0e, 0xe6, 0xe7, 0x96, 0x73, 0x49, 0xe1, 0x2e, 0xcd, + 0xa0, 0x5d, 0x40, 0xf4, 0x2a, 0xad, 0x45, 0xce, 0x36, 0x89, 0x1b, 0x1f, 0xee, 0xbf, 0xf1, 0x33, + 0x0f, 0x0e, 0xe6, 0xd1, 0x7a, 0x8a, 0x04, 0xce, 0x20, 0x8b, 0xde, 0x82, 0x53, 0xb4, 0x34, 0xf5, + 0xad, 0x23, 0xfd, 0x37, 0x37, 0xfb, 0xe0, 0x60, 0xfe, 0xd4, 0x7a, 0x06, 0x11, 0x9c, 0x49, 0x1a, + 0x7d, 0x87, 0x05, 0x67, 0xe3, 0xcf, 0x5f, 0xb9, 0xdf, 0x76, 0xbc, 0x46, 0xdc, 0x70, 0xa9, 0xff, + 0x86, 0xe9, 0x99, 0x7c, 0x76, 0x39, 0x8f, 0x12, 0xce, 0x6f, 0x64, 0x6e, 0x19, 0x4e, 0x67, 0xae, + 0x16, 0x34, 0x05, 0xc5, 0x5d, 0xc2, 0xb9, 0xa0, 0x12, 0xa6, 0x3f, 0xd1, 0x29, 0x18, 0xdc, 0x73, + 0x9a, 0x1d, 0xb1, 0x51, 0x30, 0xff, 0xf3, 0x4a, 0xe1, 0x65, 0xcb, 0xfe, 0xe7, 0x45, 0x98, 0x5c, + 0xae, 0x55, 0x1e, 0x6a, 0x17, 0xea, 0xd7, 0x50, 0xa1, 0xeb, 0x35, 0x14, 0x5f, 0x6a, 0xc5, 0xdc, + 0x4b, 0xed, 0xff, 0xc9, 0xd8, 0x42, 0x03, 0x6c, 0x0b, 0x7d, 0x4b, 0xce, 0x16, 0x3a, 0xe6, 0x8d, + 0xb3, 0x97, 0xb3, 0x8a, 0x06, 0xd9, 0x64, 0x66, 0x72, 0x2c, 0x37, 0xfd, 0xba, 0xd3, 0x4c, 0x1e, + 0x7d, 0x47, 0x5c, 0x4a, 0xc7, 0x33, 0x8f, 0x75, 0x18, 0x5b, 0x76, 0xda, 0xce, 0xa6, 0xdb, 0x74, + 0x23, 0x97, 0x84, 0xe8, 0x49, 0x28, 0x3a, 0x8d, 0x06, 0xe3, 0xb6, 0x4a, 0x4b, 0xa7, 0x1f, 0x1c, + 0xcc, 0x17, 0x17, 0x1b, 0xf4, 0xda, 0x07, 0x85, 0xb5, 0x8f, 0x29, 0x06, 0xfa, 0x20, 0x0c, 0x34, + 0x02, 0xbf, 0x3d, 0x5b, 0x60, 0x98, 0x74, 0xd7, 0x0d, 0x94, 0x03, 0xbf, 0x9d, 0x40, 0x65, 0x38, + 0xf6, 0xaf, 0x15, 0xe0, 0xdc, 0x32, 0x69, 0xef, 0xac, 0xd6, 0x72, 0xce, 0xef, 0x2b, 0x30, 0xd2, + 0xf2, 0x3d, 0x37, 0xf2, 0x83, 0x50, 0x34, 0xcd, 0x56, 0xc4, 0x9a, 0x28, 0xc3, 0x0a, 0x8a, 0x2e, + 0xc2, 0x40, 0x3b, 0x66, 0x2a, 0xc7, 0x24, 0x43, 0xca, 0xd8, 0x49, 0x06, 0xa1, 0x18, 0x9d, 0x90, + 0x04, 0x62, 0xc5, 0x28, 0x8c, 0xdb, 0x21, 0x09, 0x30, 0x83, 0xc4, 0x37, 0x33, 0xbd, 0xb3, 0xc5, + 0x09, 0x9d, 0xb8, 0x99, 0x29, 0x04, 0x6b, 0x58, 0xa8, 0x0a, 0xa5, 0x30, 0x31, 0xb3, 0x7d, 0x6d, + 0xd3, 0x71, 0x76, 0x75, 0xab, 0x99, 0x8c, 0x89, 0x18, 0x37, 0xca, 0x50, 0xcf, 0xab, 0xfb, 0x2b, + 0x05, 0x40, 0x7c, 0x08, 0xbf, 0xc1, 0x06, 0xee, 0x76, 0x7a, 0xe0, 0xfa, 0xdf, 0x12, 0xc7, 0x35, + 0x7a, 0x7f, 0x6e, 0xc1, 0xb9, 0x65, 0xd7, 0x6b, 0x90, 0x20, 0x67, 0x01, 0x3e, 0x9a, 0xb7, 0xec, + 0xd1, 0x98, 0x06, 0x63, 0x89, 0x0d, 0x1c, 0xc3, 0x12, 0xb3, 0xff, 0xc4, 0x02, 0xc4, 0x3f, 0xfb, + 0x5d, 0xf7, 0xb1, 0xb7, 0xd3, 0x1f, 0x7b, 0x0c, 0xcb, 0xc2, 0xbe, 0x09, 0x13, 0xcb, 0x4d, 0x97, + 0x78, 0x51, 0xa5, 0xba, 0xec, 0x7b, 0x5b, 0xee, 0x36, 0x7a, 0x05, 0x26, 0x22, 0xb7, 0x45, 0xfc, + 0x4e, 0x54, 0x23, 0x75, 0xdf, 0x63, 0x2f, 0x49, 0xeb, 0xca, 0xe0, 0x12, 0x7a, 0x70, 0x30, 0x3f, + 0xb1, 0x61, 0x40, 0x70, 0x02, 0xd3, 0xfe, 0x3d, 0x3a, 0x7e, 0x7e, 0xab, 0xed, 0x7b, 0xc4, 0x8b, + 0x96, 0x7d, 0xaf, 0xc1, 0x25, 0x0e, 0xaf, 0xc0, 0x40, 0x44, 0xc7, 0x83, 0x8f, 0xdd, 0x65, 0xb9, + 0x51, 0xe8, 0x28, 0x1c, 0x1e, 0xcc, 0x9f, 0x49, 0xd7, 0x60, 0xe3, 0xc4, 0xea, 0xa0, 0x6f, 0x81, + 0xa1, 0x30, 0x72, 0xa2, 0x4e, 0x28, 0x46, 0xf3, 0x09, 0x39, 0x9a, 0x35, 0x56, 0x7a, 0x78, 0x30, + 0x3f, 0xa9, 0xaa, 0xf1, 0x22, 0x2c, 0x2a, 0xa0, 0xa7, 0x60, 0xb8, 0x45, 0xc2, 0xd0, 0xd9, 0x96, + 0xb7, 0xe1, 0xa4, 0xa8, 0x3b, 0xbc, 0xc6, 0x8b, 0xb1, 0x84, 0xa3, 0x4b, 0x30, 0x48, 0x82, 0xc0, + 0x0f, 0xc4, 0x1e, 0x1d, 0x17, 0x88, 0x83, 0x2b, 0xb4, 0x10, 0x73, 0x98, 0xfd, 0x6f, 0x2c, 0x98, + 0x54, 0x7d, 0xe5, 0x6d, 0x9d, 0xc0, 0xab, 0xe0, 0x53, 0x00, 0x75, 0xf9, 0x81, 0x21, 0xbb, 0x3d, + 0x46, 0x9f, 0xbf, 0x9c, 0x79, 0x51, 0xa7, 0x86, 0x31, 0xa6, 0xac, 0x8a, 0x42, 0xac, 0x51, 0xb3, + 0xff, 0xb1, 0x05, 0x33, 0x89, 0x2f, 0xba, 0xe9, 0x86, 0x11, 0x7a, 0x33, 0xf5, 0x55, 0x0b, 0xfd, + 0x7d, 0x15, 0xad, 0xcd, 0xbe, 0x49, 0x2d, 0x65, 0x59, 0xa2, 0x7d, 0xd1, 0x75, 0x18, 0x74, 0x23, + 0xd2, 0x92, 0x1f, 0x73, 0xa9, 0xeb, 0xc7, 0xf0, 0x5e, 0xc5, 0x33, 0x52, 0xa1, 0x35, 0x31, 0x27, + 0x60, 0xff, 0x5a, 0x11, 0x4a, 0x7c, 0xd9, 0xae, 0x39, 0xed, 0x13, 0x98, 0x8b, 0xa7, 0xa1, 0xe4, + 0xb6, 0x5a, 0x9d, 0xc8, 0xd9, 0x14, 0xc7, 0xf9, 0x08, 0xdf, 0x5a, 0x15, 0x59, 0x88, 0x63, 0x38, + 0xaa, 0xc0, 0x00, 0xeb, 0x0a, 0xff, 0xca, 0x27, 0xb3, 0xbf, 0x52, 0xf4, 0x7d, 0xa1, 0xec, 0x44, + 0x0e, 0xe7, 0xa4, 0xd4, 0x3d, 0x42, 0x8b, 0x30, 0x23, 0x81, 0x1c, 0x80, 0x4d, 0xd7, 0x73, 0x82, + 0x7d, 0x5a, 0x36, 0x5b, 0x64, 0x04, 0x9f, 0xed, 0x4e, 0x70, 0x49, 0xe1, 0x73, 0xb2, 0xea, 0xc3, + 0x62, 0x00, 0xd6, 0x88, 0xce, 0x7d, 0x04, 0x4a, 0x0a, 0xf9, 0x28, 0x0c, 0xd1, 0xdc, 0xc7, 0x60, + 0x32, 0xd1, 0x56, 0xaf, 0xea, 0x63, 0x3a, 0x3f, 0xf5, 0xcb, 0xec, 0xc8, 0x10, 0xbd, 0x5e, 0xf1, + 0xf6, 0xc4, 0x91, 0xfb, 0x36, 0x9c, 0x6a, 0x66, 0x9c, 0x64, 0x62, 0x5e, 0xfb, 0x3f, 0xf9, 0xce, + 0x89, 0xcf, 0x3e, 0x95, 0x05, 0xc5, 0x99, 0x6d, 0x50, 0x1e, 0xc1, 0x6f, 0xd3, 0x0d, 0xe2, 0x34, + 0x75, 0x76, 0xfb, 0x96, 0x28, 0xc3, 0x0a, 0x4a, 0xcf, 0xbb, 0x53, 0xaa, 0xf3, 0x37, 0xc8, 0x7e, + 0x8d, 0x34, 0x49, 0x3d, 0xf2, 0x83, 0xaf, 0x6b, 0xf7, 0xcf, 0xf3, 0xd1, 0xe7, 0xc7, 0xe5, 0xa8, + 0x20, 0x50, 0xbc, 0x41, 0xf6, 0xf9, 0x54, 0xe8, 0x5f, 0x57, 0xec, 0xfa, 0x75, 0x3f, 0x6b, 0xc1, + 0xb8, 0xfa, 0xba, 0x13, 0x38, 0x17, 0x96, 0xcc, 0x73, 0xe1, 0x7c, 0xd7, 0x05, 0x9e, 0x73, 0x22, + 0x7c, 0xa5, 0x00, 0x67, 0x15, 0x0e, 0x7d, 0x1b, 0xf0, 0x3f, 0x62, 0x55, 0x5d, 0x85, 0x92, 0xa7, + 0xa4, 0x56, 0x96, 0x29, 0x2e, 0x8a, 0x65, 0x56, 0x31, 0x0e, 0x65, 0xf1, 0xbc, 0x58, 0xb4, 0x34, + 0xa6, 0x8b, 0x73, 0x85, 0xe8, 0x76, 0x09, 0x8a, 0x1d, 0xb7, 0x21, 0x2e, 0x98, 0x0f, 0xc9, 0xd1, + 0xbe, 0x5d, 0x29, 0x1f, 0x1e, 0xcc, 0x3f, 0x91, 0xa7, 0x4a, 0xa0, 0x37, 0x5b, 0xb8, 0x70, 0xbb, + 0x52, 0xc6, 0xb4, 0x32, 0x5a, 0x84, 0x49, 0xa9, 0x2d, 0xb9, 0x43, 0xd9, 0x2d, 0xdf, 0x13, 0xf7, + 0x90, 0x92, 0xc9, 0x62, 0x13, 0x8c, 0x93, 0xf8, 0xa8, 0x0c, 0x53, 0xbb, 0x9d, 0x4d, 0xd2, 0x24, + 0x11, 0xff, 0xe0, 0x1b, 0x84, 0x4b, 0x2c, 0x4b, 0xf1, 0xcb, 0xec, 0x46, 0x02, 0x8e, 0x53, 0x35, + 0xec, 0xbf, 0x64, 0xf7, 0x81, 0x18, 0xbd, 0x6a, 0xe0, 0xd3, 0x85, 0x45, 0xa9, 0x7f, 0x3d, 0x97, + 0x73, 0x3f, 0xab, 0xe2, 0x06, 0xd9, 0xdf, 0xf0, 0x29, 0x67, 0x9e, 0xbd, 0x2a, 0x8c, 0x35, 0x3f, + 0xd0, 0x75, 0xcd, 0xff, 0x7c, 0x01, 0x4e, 0xab, 0x11, 0x30, 0x98, 0xc0, 0x6f, 0xf4, 0x31, 0x78, + 0x0e, 0x46, 0x1b, 0x64, 0xcb, 0xe9, 0x34, 0x23, 0x25, 0x3e, 0x1f, 0xe4, 0x2a, 0x94, 0x72, 0x5c, + 0x8c, 0x75, 0x9c, 0x23, 0x0c, 0xdb, 0xff, 0x18, 0x65, 0x17, 0x71, 0xe4, 0xd0, 0x35, 0xae, 0x76, + 0x8d, 0x95, 0xbb, 0x6b, 0x2e, 0xc1, 0xa0, 0xdb, 0xa2, 0x8c, 0x59, 0xc1, 0xe4, 0xb7, 0x2a, 0xb4, + 0x10, 0x73, 0x18, 0xfa, 0x00, 0x0c, 0xd7, 0xfd, 0x56, 0xcb, 0xf1, 0x1a, 0xec, 0xca, 0x2b, 0x2d, + 0x8d, 0x52, 0xde, 0x6d, 0x99, 0x17, 0x61, 0x09, 0x43, 0xe7, 0x60, 0xc0, 0x09, 0xb6, 0xb9, 0x0c, + 0xa3, 0xb4, 0x34, 0x42, 0x5b, 0x5a, 0x0c, 0xb6, 0x43, 0xcc, 0x4a, 0xe9, 0x13, 0xec, 0x9e, 0x1f, + 0xec, 0xba, 0xde, 0x76, 0xd9, 0x0d, 0xc4, 0x96, 0x50, 0x77, 0xe1, 0x5d, 0x05, 0xc1, 0x1a, 0x16, + 0x5a, 0x85, 0xc1, 0xb6, 0x1f, 0x44, 0xe1, 0xec, 0x10, 0x1b, 0xee, 0x27, 0x72, 0x0e, 0x22, 0xfe, + 0xb5, 0x55, 0x3f, 0x88, 0xe2, 0x0f, 0xa0, 0xff, 0x42, 0xcc, 0xab, 0xa3, 0x9b, 0x30, 0x4c, 0xbc, + 0xbd, 0xd5, 0xc0, 0x6f, 0xcd, 0xce, 0xe4, 0x53, 0x5a, 0xe1, 0x28, 0x7c, 0x99, 0xc5, 0x3c, 0xaa, + 0x28, 0xc6, 0x92, 0x04, 0xfa, 0x16, 0x28, 0x12, 0x6f, 0x6f, 0x76, 0x98, 0x51, 0x9a, 0xcb, 0xa1, + 0x74, 0xc7, 0x09, 0xe2, 0x33, 0x7f, 0xc5, 0xdb, 0xc3, 0xb4, 0x0e, 0xfa, 0x24, 0x94, 0xe4, 0x81, + 0x11, 0x0a, 0x61, 0x5d, 0xe6, 0x82, 0x95, 0xc7, 0x0c, 0x26, 0x6f, 0x75, 0xdc, 0x80, 0xb4, 0x88, + 0x17, 0x85, 0xf1, 0x09, 0x29, 0xa1, 0x21, 0x8e, 0xa9, 0xa1, 0x4f, 0x4a, 0x09, 0xf1, 0x9a, 0xdf, + 0xf1, 0xa2, 0x70, 0xb6, 0xc4, 0xba, 0x97, 0xa9, 0xbb, 0xbb, 0x13, 0xe3, 0x25, 0x45, 0xc8, 0xbc, + 0x32, 0x36, 0x48, 0xa1, 0x4f, 0xc3, 0x38, 0xff, 0xcf, 0x35, 0x60, 0xe1, 0xec, 0x69, 0x46, 0xfb, + 0x62, 0x3e, 0x6d, 0x8e, 0xb8, 0x74, 0x5a, 0x10, 0x1f, 0xd7, 0x4b, 0x43, 0x6c, 0x52, 0x43, 0x18, + 0xc6, 0x9b, 0xee, 0x1e, 0xf1, 0x48, 0x18, 0x56, 0x03, 0x7f, 0x93, 0xcc, 0x02, 0x1b, 0x98, 0xb3, + 0xd9, 0x1a, 0x33, 0x7f, 0x93, 0x2c, 0x4d, 0x53, 0x9a, 0x37, 0xf5, 0x3a, 0xd8, 0x24, 0x81, 0x6e, + 0xc3, 0x04, 0x7d, 0xb1, 0xb9, 0x31, 0xd1, 0xd1, 0x5e, 0x44, 0xd9, 0xbb, 0x0a, 0x1b, 0x95, 0x70, + 0x82, 0x08, 0xba, 0x05, 0x63, 0x61, 0xe4, 0x04, 0x51, 0xa7, 0xcd, 0x89, 0x9e, 0xe9, 0x45, 0x94, + 0x29, 0x5c, 0x6b, 0x5a, 0x15, 0x6c, 0x10, 0x40, 0xaf, 0x43, 0xa9, 0xe9, 0x6e, 0x91, 0xfa, 0x7e, + 0xbd, 0x49, 0x66, 0xc7, 0x18, 0xb5, 0xcc, 0x43, 0xe5, 0xa6, 0x44, 0xe2, 0x7c, 0xae, 0xfa, 0x8b, + 0xe3, 0xea, 0xe8, 0x0e, 0x9c, 0x89, 0x48, 0xd0, 0x72, 0x3d, 0x87, 0x1e, 0x06, 0xe2, 0x69, 0xc5, + 0x14, 0x99, 0xe3, 0x6c, 0xb7, 0x5d, 0x10, 0xb3, 0x71, 0x66, 0x23, 0x13, 0x0b, 0xe7, 0xd4, 0x46, + 0xf7, 0x61, 0x36, 0x03, 0xe2, 0x37, 0xdd, 0xfa, 0xfe, 0xec, 0x29, 0x46, 0xf9, 0xa3, 0x82, 0xf2, + 0xec, 0x46, 0x0e, 0xde, 0x61, 0x17, 0x18, 0xce, 0xa5, 0x8e, 0x6e, 0xc1, 0x24, 0x3b, 0x81, 0xaa, + 0x9d, 0x66, 0x53, 0x34, 0x38, 0xc1, 0x1a, 0xfc, 0x80, 0xbc, 0x8f, 0x2b, 0x26, 0xf8, 0xf0, 0x60, + 0x1e, 0xe2, 0x7f, 0x38, 0x59, 0x1b, 0x6d, 0x32, 0x9d, 0x59, 0x27, 0x70, 0xa3, 0x7d, 0x7a, 0x6e, + 0x90, 0xfb, 0xd1, 0xec, 0x64, 0x57, 0x79, 0x85, 0x8e, 0xaa, 0x14, 0x6b, 0x7a, 0x21, 0x4e, 0x12, + 0xa4, 0x47, 0x6a, 0x18, 0x35, 0x5c, 0x6f, 0x76, 0x8a, 0xbf, 0x4b, 0xe4, 0x89, 0x54, 0xa3, 0x85, + 0x98, 0xc3, 0x98, 0xbe, 0x8c, 0xfe, 0xb8, 0x45, 0x6f, 0xae, 0x69, 0x86, 0x18, 0xeb, 0xcb, 0x24, + 0x00, 0xc7, 0x38, 0x94, 0x99, 0x8c, 0xa2, 0xfd, 0x59, 0xc4, 0x50, 0xd5, 0xc1, 0xb2, 0xb1, 0xf1, + 0x49, 0x4c, 0xcb, 0xed, 0x4d, 0x98, 0x50, 0x07, 0x21, 0x1b, 0x13, 0x34, 0x0f, 0x83, 0x8c, 0x7d, + 0x12, 0xd2, 0xb5, 0x12, 0xed, 0x02, 0x63, 0xad, 0x30, 0x2f, 0x67, 0x5d, 0x70, 0xdf, 0x26, 0x4b, + 0xfb, 0x11, 0xe1, 0x6f, 0xfa, 0xa2, 0xd6, 0x05, 0x09, 0xc0, 0x31, 0x8e, 0xfd, 0xbf, 0x39, 0x1b, + 0x1a, 0x9f, 0xb6, 0x7d, 0xdc, 0x2f, 0xcf, 0xc0, 0xc8, 0x8e, 0x1f, 0x46, 0x14, 0x9b, 0xb5, 0x31, + 0x18, 0x33, 0x9e, 0xd7, 0x45, 0x39, 0x56, 0x18, 0xe8, 0x55, 0x18, 0xaf, 0xeb, 0x0d, 0x88, 0xcb, + 0x51, 0x1d, 0x23, 0x46, 0xeb, 0xd8, 0xc4, 0x45, 0x2f, 0xc3, 0x08, 0xb3, 0x01, 0xa9, 0xfb, 0x4d, + 0xc1, 0xb5, 0xc9, 0x1b, 0x7e, 0xa4, 0x2a, 0xca, 0x0f, 0xb5, 0xdf, 0x58, 0x61, 0xa3, 0xcb, 0x30, + 0x44, 0xbb, 0x50, 0xa9, 0x8a, 0x6b, 0x49, 0x09, 0x8a, 0xae, 0xb3, 0x52, 0x2c, 0xa0, 0xf6, 0xff, + 0x5f, 0xd0, 0x46, 0x99, 0xbe, 0x87, 0x09, 0xaa, 0xc2, 0xf0, 0x3d, 0xc7, 0x8d, 0x5c, 0x6f, 0x5b, + 0xf0, 0x1f, 0x4f, 0x75, 0xbd, 0xa3, 0x58, 0xa5, 0xbb, 0xbc, 0x02, 0xbf, 0x45, 0xc5, 0x1f, 0x2c, + 0xc9, 0x50, 0x8a, 0x41, 0xc7, 0xf3, 0x28, 0xc5, 0x42, 0xbf, 0x14, 0x31, 0xaf, 0xc0, 0x29, 0x8a, + 0x3f, 0x58, 0x92, 0x41, 0x6f, 0x02, 0xc8, 0x1d, 0x46, 0x1a, 0xc2, 0xf6, 0xe2, 0x99, 0xde, 0x44, + 0x37, 0x54, 0x9d, 0xa5, 0x09, 0x7a, 0x47, 0xc7, 0xff, 0xb1, 0x46, 0xcf, 0x8e, 0x18, 0x9f, 0x96, + 0xee, 0x0c, 0xfa, 0x36, 0xba, 0xc4, 0x9d, 0x20, 0x22, 0x8d, 0xc5, 0x48, 0x0c, 0xce, 0x07, 0xfb, + 0x7b, 0xa4, 0x6c, 0xb8, 0x2d, 0xa2, 0x6f, 0x07, 0x41, 0x04, 0xc7, 0xf4, 0xec, 0x5f, 0x2c, 0xc2, + 0x6c, 0x5e, 0x77, 0xe9, 0xa2, 0x23, 0xf7, 0xdd, 0x68, 0x99, 0xb2, 0x57, 0x96, 0xb9, 0xe8, 0x56, + 0x44, 0x39, 0x56, 0x18, 0x74, 0xf6, 0x43, 0x77, 0x5b, 0xbe, 0x31, 0x07, 0xe3, 0xd9, 0xaf, 0xb1, + 0x52, 0x2c, 0xa0, 0x14, 0x2f, 0x20, 0x4e, 0x28, 0x8c, 0x7b, 0xb4, 0x55, 0x82, 0x59, 0x29, 0x16, + 0x50, 0x5d, 0xda, 0x35, 0xd0, 0x43, 0xda, 0x65, 0x0c, 0xd1, 0xe0, 0xf1, 0x0e, 0x11, 0xfa, 0x0c, + 0xc0, 0x96, 0xeb, 0xb9, 0xe1, 0x0e, 0xa3, 0x3e, 0x74, 0x64, 0xea, 0x8a, 0x39, 0x5b, 0x55, 0x54, + 0xb0, 0x46, 0x11, 0xbd, 0x04, 0xa3, 0x6a, 0x03, 0x56, 0xca, 0x4c, 0xd3, 0xa9, 0x59, 0x8e, 0xc4, + 0xa7, 0x51, 0x19, 0xeb, 0x78, 0xf6, 0xe7, 0x92, 0xeb, 0x45, 0xec, 0x00, 0x6d, 0x7c, 0xad, 0x7e, + 0xc7, 0xb7, 0xd0, 0x7d, 0x7c, 0xed, 0xaf, 0x15, 0x61, 0xd2, 0x68, 0xac, 0x13, 0xf6, 0x71, 0x66, 0x5d, 0xa3, 0x07, 0xb8, 0x13, 0x11, 0xb1, 0xff, 0xec, 0xde, 0x5b, 0x45, 0x3f, 0xe4, 0xe9, 0x0e, - 0xe0, 0xf5, 0xd1, 0x67, 0xa1, 0xd8, 0x70, 0x42, 0x26, 0x39, 0x23, 0x62, 0xdf, 0xf5, 0x43, 0x2c, + 0xe0, 0xf5, 0xd1, 0x67, 0xa0, 0xd4, 0x74, 0x42, 0x26, 0x39, 0x23, 0x62, 0xdf, 0xf5, 0x43, 0x2c, 0x7e, 0x98, 0x38, 0x61, 0xa4, 0xdd, 0x9a, 0x9c, 0x76, 0x4c, 0x92, 0xde, 0x34, 0x94, 0x3f, 0x91, - 0xd6, 0x63, 0xaa, 0x13, 0x94, 0x89, 0xd9, 0xc3, 0x1c, 0x86, 0x5e, 0x82, 0xd1, 0x80, 0xb0, 0x55, - 0xb1, 0x44, 0xb9, 0x39, 0xb6, 0xcc, 0x06, 0x63, 0xb6, 0x0f, 0x6b, 0x30, 0x6c, 0x60, 0xc6, 0x6f, - 0x83, 0xa1, 0x2e, 0x6f, 0x83, 0x27, 0x60, 0x98, 0xfd, 0x50, 0x2b, 0x40, 0xcd, 0x46, 0x99, 0x17, - 0x63, 0x09, 0x4f, 0x2e, 0x98, 0x42, 0x7f, 0x0b, 0x86, 0xbe, 0x3e, 0xc4, 0xa2, 0x66, 0x5a, 0xe6, - 0x02, 0x3f, 0xe5, 0xc4, 0x92, 0xc7, 0x12, 0x66, 0x7f, 0x14, 0xc6, 0x4b, 0x0e, 0x69, 0xfa, 0xde, - 0xb2, 0x57, 0x6f, 0xf9, 0xae, 0x17, 0xa1, 0x19, 0x18, 0x60, 0x97, 0x08, 0x3f, 0x02, 0x06, 0x68, - 0x43, 0x78, 0x80, 0x3e, 0x08, 0xec, 0x2d, 0x38, 0x5d, 0xf2, 0xef, 0x79, 0xf7, 0x9c, 0xa0, 0xbe, - 0x50, 0x29, 0x6b, 0xef, 0xeb, 0x35, 0xf9, 0xbe, 0xe3, 0x46, 0x5b, 0xa9, 0x47, 0xaf, 0x56, 0x93, - 0xb3, 0xb5, 0x2b, 0x6e, 0x83, 0x64, 0x48, 0x41, 0xfe, 0x6a, 0xce, 0x68, 0x29, 0xc6, 0x57, 0x5a, - 0x2d, 0x2b, 0x53, 0xab, 0xf5, 0x06, 0x14, 0x36, 0x5d, 0xd2, 0xa8, 0x63, 0xb2, 0x29, 0x56, 0xe2, - 0xe3, 0xd9, 0x76, 0x28, 0x2b, 0x14, 0x53, 0x4a, 0xbd, 0xf8, 0xeb, 0x70, 0x45, 0x54, 0xc6, 0x8a, - 0x0c, 0xda, 0x81, 0x49, 0xf9, 0x60, 0x90, 0x50, 0xb1, 0x2e, 0x9f, 0xe8, 0xf6, 0x0a, 0x31, 0x89, - 0x9f, 0x7a, 0xb0, 0x3f, 0x37, 0x89, 0x13, 0x64, 0x70, 0x07, 0x61, 0xfa, 0x1c, 0x6c, 0xd2, 0x13, - 0x78, 0x80, 0x0d, 0x3f, 0x7b, 0x0e, 0xb2, 0x97, 0x2d, 0x2b, 0xb5, 0x7f, 0xcc, 0x82, 0x47, 0x3a, - 0x46, 0x46, 0xbc, 0xf0, 0x8f, 0x78, 0x16, 0x92, 0x2f, 0xee, 0x5c, 0xef, 0x17, 0xb7, 0xfd, 0x77, - 0x2c, 0x38, 0xb5, 0xdc, 0x6c, 0x45, 0x7b, 0x25, 0xd7, 0x54, 0x41, 0xbd, 0x08, 0x43, 0x4d, 0x52, - 0x77, 0xdb, 0x4d, 0x31, 0x73, 0x73, 0xf2, 0x94, 0x5a, 0x65, 0xa5, 0x07, 0xfb, 0x73, 0x63, 0xd5, - 0xc8, 0x0f, 0x9c, 0x2d, 0xc2, 0x0b, 0xb0, 0x40, 0x67, 0x67, 0xbd, 0xfb, 0x2e, 0xb9, 0xe9, 0x36, - 0x5d, 0x69, 0x57, 0xd4, 0x55, 0x66, 0x37, 0x2f, 0x07, 0x74, 0xfe, 0x8d, 0xb6, 0xe3, 0x45, 0x6e, - 0xb4, 0x27, 0xb4, 0x47, 0x92, 0x08, 0x8e, 0xe9, 0xd9, 0x5f, 0xb5, 0x60, 0x42, 0xae, 0xfb, 0x85, - 0x7a, 0x3d, 0x20, 0x61, 0x88, 0x66, 0x21, 0xe7, 0xb6, 0x44, 0x2f, 0x41, 0xf4, 0x32, 0x57, 0xae, - 0xe0, 0x9c, 0xdb, 0x92, 0x6c, 0x19, 0x3b, 0x08, 0xf3, 0xa6, 0x22, 0xed, 0xba, 0x28, 0xc7, 0x0a, - 0x03, 0x5d, 0x81, 0x82, 0xe7, 0xd7, 0xb9, 0x6d, 0x17, 0xbf, 0xd2, 0xd8, 0x02, 0x5b, 0x13, 0x65, - 0x58, 0x41, 0x51, 0x05, 0x8a, 0xdc, 0xec, 0x29, 0x5e, 0xb4, 0x7d, 0x19, 0x4f, 0xb1, 0x2f, 0x5b, - 0x97, 0x35, 0x71, 0x4c, 0xc4, 0xfe, 0x55, 0x0b, 0x46, 0xe5, 0x97, 0xf5, 0xc9, 0x73, 0xd2, 0xad, - 0x15, 0xf3, 0x9b, 0xf1, 0xd6, 0xa2, 0x3c, 0x23, 0x83, 0x18, 0xac, 0x62, 0xfe, 0x50, 0xac, 0xe2, - 0x33, 0x30, 0xe2, 0xb4, 0x5a, 0x15, 0x93, 0xcf, 0x64, 0x4b, 0x69, 0x21, 0x2e, 0xc6, 0x3a, 0x8e, - 0xfd, 0xa3, 0x39, 0x18, 0x97, 0x5f, 0x50, 0x6d, 0x6f, 0x84, 0x24, 0x42, 0xeb, 0x50, 0x74, 0xf8, - 0x2c, 0x11, 0xb9, 0xc8, 0x2f, 0xa5, 0xcb, 0x11, 0x8c, 0x29, 0x8d, 0x2f, 0xfc, 0x05, 0x59, 0x1b, - 0xc7, 0x84, 0x50, 0x03, 0xa6, 0x3c, 0x3f, 0x62, 0x87, 0xbf, 0x82, 0x77, 0x53, 0xed, 0x24, 0xa9, - 0x9f, 0x15, 0xd4, 0xa7, 0xd6, 0x92, 0x54, 0x70, 0x27, 0x61, 0xb4, 0x2c, 0x65, 0x33, 0xf9, 0x6c, - 0x61, 0x80, 0x3e, 0x71, 0xe9, 0xa2, 0x19, 0xfb, 0x57, 0x2c, 0x28, 0x4a, 0xb4, 0x93, 0xd0, 0xe2, - 0xad, 0xc2, 0x70, 0xc8, 0x26, 0x41, 0x0e, 0x8d, 0xdd, 0xad, 0xe3, 0x7c, 0xbe, 0xe2, 0x3b, 0x8d, - 0xff, 0x0f, 0xb1, 0xa4, 0xc1, 0x44, 0xf3, 0xaa, 0xfb, 0xef, 0x13, 0xd1, 0xbc, 0xea, 0x4f, 0xc6, - 0xa5, 0xf4, 0x87, 0xac, 0xcf, 0x9a, 0xac, 0x8b, 0xb2, 0x5e, 0xad, 0x80, 0x6c, 0xba, 0xf7, 0x93, - 0xac, 0x57, 0x85, 0x95, 0x62, 0x01, 0x45, 0x6f, 0xc1, 0x68, 0x4d, 0xca, 0x64, 0xe3, 0x1d, 0x7e, - 0xb9, 0xab, 0x7e, 0x40, 0xa9, 0x92, 0xb8, 0x2c, 0x64, 0x49, 0xab, 0x8f, 0x0d, 0x6a, 0xa6, 0x19, - 0x41, 0xbe, 0x97, 0x19, 0x41, 0x4c, 0x37, 0x5b, 0xa9, 0xfe, 0xe3, 0x16, 0x0c, 0x71, 0x59, 0x5c, - 0x7f, 0xa2, 0x50, 0x4d, 0xb3, 0x16, 0x8f, 0xdd, 0x1d, 0x5a, 0x28, 0x34, 0x65, 0x68, 0x15, 0x8a, - 0xec, 0x07, 0x93, 0x25, 0xe6, 0xb3, 0xad, 0xee, 0x79, 0xab, 0x7a, 0x07, 0xef, 0xc8, 0x6a, 0x38, - 0xa6, 0x60, 0xff, 0x70, 0x9e, 0x9e, 0x6e, 0x31, 0xaa, 0x71, 0xe9, 0x5b, 0xc7, 0x77, 0xe9, 0xe7, - 0x8e, 0xeb, 0xd2, 0xdf, 0x82, 0x89, 0x9a, 0xa6, 0x87, 0x8b, 0x67, 0xf2, 0x4a, 0xd7, 0x45, 0xa2, - 0xa9, 0xec, 0xb8, 0x94, 0x65, 0xc9, 0x24, 0x82, 0x93, 0x54, 0xd1, 0xb7, 0xc1, 0x28, 0x9f, 0x67, - 0xd1, 0x0a, 0xb7, 0xc4, 0xf8, 0x48, 0xf6, 0x7a, 0xd1, 0x9b, 0xe0, 0x52, 0x39, 0xad, 0x3a, 0x36, - 0x88, 0xd9, 0x7f, 0x62, 0x01, 0x5a, 0x6e, 0x6d, 0x93, 0x26, 0x09, 0x9c, 0x46, 0x2c, 0x4e, 0xff, - 0x7e, 0x0b, 0x66, 0x48, 0x47, 0xf1, 0x92, 0xdf, 0x6c, 0x8a, 0x47, 0x4b, 0xc6, 0xbb, 0x7a, 0x39, - 0xa3, 0x8e, 0x72, 0x4b, 0x98, 0xc9, 0xc2, 0xc0, 0x99, 0xed, 0xa1, 0x55, 0x98, 0xe6, 0xb7, 0xa4, - 0x02, 0x68, 0xb6, 0xd7, 0x8f, 0x0a, 0xc2, 0xd3, 0xeb, 0x9d, 0x28, 0x38, 0xad, 0x9e, 0xfd, 0x5d, - 0xa3, 0x90, 0xd9, 0x8b, 0x0f, 0xf4, 0x08, 0x1f, 0xe8, 0x11, 0x3e, 0xd0, 0x23, 0x7c, 0xa0, 0x47, - 0xf8, 0x40, 0x8f, 0xf0, 0x4d, 0xaf, 0x47, 0xf8, 0x23, 0x0b, 0xa6, 0x3b, 0xaf, 0x81, 0x93, 0x60, - 0xcc, 0xdb, 0x30, 0xdd, 0x79, 0xd7, 0x75, 0xb5, 0xb3, 0xeb, 0xec, 0x67, 0x7c, 0xef, 0xa5, 0x7c, - 0x03, 0x4e, 0xa3, 0x6f, 0xff, 0xba, 0x05, 0xa7, 0x15, 0xb2, 0xf1, 0xd2, 0xff, 0x3c, 0x4c, 0xf3, - 0xf3, 0x65, 0xa9, 0xe1, 0xb8, 0xcd, 0x75, 0xd2, 0x6c, 0x35, 0x9c, 0x48, 0x9a, 0x19, 0x3c, 0x93, - 0xba, 0x55, 0x13, 0x26, 0xba, 0x46, 0xc5, 0xc5, 0x47, 0x68, 0xbf, 0x52, 0x00, 0x38, 0xad, 0x19, - 0xc3, 0x28, 0x35, 0xd7, 0xd3, 0x4c, 0xf8, 0x17, 0x0b, 0x30, 0xb8, 0xbc, 0x4b, 0xbc, 0xe8, 0x04, - 0x26, 0xaa, 0x06, 0xe3, 0xae, 0xb7, 0xeb, 0x37, 0x76, 0x49, 0x9d, 0xc3, 0x0f, 0xf3, 0xd0, 0x3f, - 0x23, 0x48, 0x8f, 0x97, 0x0d, 0x12, 0x38, 0x41, 0xf2, 0x38, 0x84, 0xed, 0xd7, 0x60, 0x88, 0xdf, - 0x71, 0x42, 0xd2, 0x9e, 0x7a, 0xa5, 0xb1, 0x41, 0x14, 0x37, 0x77, 0xac, 0x08, 0xe0, 0x77, 0xa8, - 0xa8, 0x8e, 0xde, 0x86, 0xf1, 0x4d, 0x37, 0x08, 0xa3, 0x75, 0xb7, 0x49, 0xc2, 0xc8, 0x69, 0xb6, - 0x1e, 0x42, 0xb8, 0xae, 0xc6, 0x61, 0xc5, 0xa0, 0x84, 0x13, 0x94, 0xd1, 0x16, 0x8c, 0x35, 0x1c, - 0xbd, 0xa9, 0xe1, 0x43, 0x37, 0xa5, 0x2e, 0xcf, 0x9b, 0x3a, 0x21, 0x6c, 0xd2, 0xa5, 0xa7, 0x4d, - 0x8d, 0xc9, 0x87, 0x0b, 0x4c, 0x6a, 0xa2, 0x4e, 0x1b, 0x2e, 0x18, 0xe6, 0x30, 0xca, 0x07, 0x32, - 0xfb, 0xe1, 0xa2, 0xc9, 0x07, 0x6a, 0x56, 0xc2, 0x9f, 0x83, 0x22, 0xa1, 0x43, 0x48, 0x09, 0x8b, - 0xfb, 0xf7, 0x6a, 0x7f, 0x7d, 0x5d, 0x75, 0x6b, 0x81, 0x6f, 0xaa, 0x35, 0x96, 0x25, 0x25, 0x1c, - 0x13, 0x45, 0x4b, 0x30, 0x14, 0x92, 0xc0, 0x25, 0xa1, 0xb8, 0x89, 0xbb, 0x4c, 0x23, 0x43, 0xe3, - 0xae, 0x37, 0xfc, 0x37, 0x16, 0x55, 0xe9, 0xf2, 0x72, 0x98, 0xc4, 0x97, 0xdd, 0x95, 0xda, 0xf2, - 0x5a, 0x60, 0xa5, 0x58, 0x40, 0xd1, 0xeb, 0x30, 0x1c, 0x90, 0x06, 0xd3, 0x9b, 0x8d, 0xf5, 0xbf, - 0xc8, 0xb9, 0x1a, 0x8e, 0xd7, 0xc3, 0x92, 0x00, 0xba, 0x01, 0x28, 0x20, 0x94, 0x8f, 0x74, 0xbd, - 0x2d, 0x65, 0x55, 0x2b, 0xee, 0x21, 0x75, 0x6e, 0xe1, 0x18, 0x43, 0x7a, 0x41, 0xe1, 0x94, 0x6a, - 0xe8, 0x1a, 0x4c, 0xa9, 0xd2, 0xb2, 0x17, 0x46, 0x0e, 0x3d, 0xff, 0x27, 0x18, 0x2d, 0x25, 0xc6, - 0xc1, 0x49, 0x04, 0xdc, 0x59, 0xc7, 0xfe, 0xb2, 0x05, 0x7c, 0x9c, 0x4f, 0x40, 0x78, 0xf1, 0x9a, - 0x29, 0xbc, 0x38, 0x9b, 0x39, 0x73, 0x19, 0x82, 0x8b, 0x2f, 0x5b, 0x30, 0xa2, 0xcd, 0x6c, 0xbc, - 0x66, 0xad, 0x2e, 0x6b, 0xb6, 0x0d, 0x93, 0x74, 0xa5, 0xdf, 0xda, 0x08, 0x49, 0xb0, 0x4b, 0xea, - 0x6c, 0x61, 0xe6, 0x1e, 0x6e, 0x61, 0x2a, 0x0b, 0xbe, 0x9b, 0x09, 0x82, 0xb8, 0xa3, 0x09, 0xfb, - 0x73, 0xb2, 0xab, 0xca, 0xe0, 0xb1, 0xa6, 0xe6, 0x3c, 0x61, 0xf0, 0xa8, 0x66, 0x15, 0xc7, 0x38, - 0x74, 0xab, 0x6d, 0xfb, 0x61, 0x94, 0x34, 0x78, 0xbc, 0xee, 0x87, 0x11, 0x66, 0x10, 0xfb, 0x39, - 0x80, 0xe5, 0xfb, 0xa4, 0xc6, 0x57, 0xac, 0xfe, 0xb6, 0xb2, 0xb2, 0xdf, 0x56, 0xf6, 0x6f, 0x5b, - 0x30, 0xbe, 0xb2, 0x64, 0xdc, 0x73, 0xf3, 0x00, 0xfc, 0x41, 0x78, 0xf7, 0xee, 0x9a, 0xb4, 0x16, - 0xe0, 0x0a, 0x5f, 0x55, 0x8a, 0x35, 0x0c, 0x74, 0x16, 0xf2, 0x8d, 0xb6, 0x27, 0xa4, 0xab, 0xc3, - 0x94, 0x7b, 0xb8, 0xd9, 0xf6, 0x30, 0x2d, 0xd3, 0x3c, 0x2e, 0xf2, 0x7d, 0x7b, 0x5c, 0xf4, 0x8c, - 0x7c, 0x80, 0xe6, 0x60, 0xf0, 0xde, 0x3d, 0xb7, 0xce, 0xfd, 0x4b, 0x85, 0x25, 0xc3, 0xdd, 0xbb, - 0xe5, 0x52, 0x88, 0x79, 0xb9, 0xfd, 0xa5, 0x3c, 0xcc, 0xae, 0x34, 0xc8, 0xfd, 0xf7, 0xe8, 0x63, - 0xdb, 0xaf, 0xbf, 0xc8, 0xe1, 0xe4, 0x54, 0x87, 0xf5, 0x09, 0xea, 0x3d, 0x1e, 0x9b, 0x30, 0xcc, - 0xed, 0xfd, 0xa4, 0xc7, 0xed, 0x2b, 0x69, 0xad, 0x67, 0x0f, 0xc8, 0x3c, 0xb7, 0x1b, 0x14, 0x0e, - 0x83, 0xea, 0xc2, 0x14, 0xa5, 0x58, 0x12, 0x9f, 0x7d, 0x19, 0x46, 0x75, 0xcc, 0x43, 0x79, 0xe7, - 0xfd, 0xff, 0x79, 0x98, 0xa4, 0x3d, 0x38, 0xd6, 0x89, 0xb8, 0xdd, 0x39, 0x11, 0x47, 0xed, 0xa1, - 0xd5, 0x7b, 0x36, 0xde, 0x4a, 0xce, 0xc6, 0x33, 0x59, 0xb3, 0x71, 0xd2, 0x73, 0xf0, 0x9d, 0x16, - 0x4c, 0xaf, 0x34, 0xfc, 0xda, 0x4e, 0xc2, 0x8b, 0xea, 0x05, 0x18, 0xa1, 0xc7, 0x71, 0x68, 0x38, - 0xf8, 0x1b, 0x21, 0x1f, 0x04, 0x08, 0xeb, 0x78, 0x5a, 0xb5, 0xdb, 0xb7, 0xcb, 0xa5, 0xb4, 0x48, - 0x11, 0x02, 0x84, 0x75, 0x3c, 0xfb, 0x37, 0x2d, 0x38, 0x7f, 0x6d, 0x69, 0x39, 0x5e, 0x8a, 0x1d, - 0xc1, 0x2a, 0x2e, 0xc3, 0x50, 0xab, 0xae, 0x75, 0x25, 0x96, 0x3e, 0x97, 0x58, 0x2f, 0x04, 0xf4, - 0xfd, 0x12, 0x88, 0xe5, 0xa7, 0x2d, 0x98, 0xbe, 0xe6, 0x46, 0xf4, 0x76, 0x4d, 0x86, 0x4d, 0xa0, - 0xd7, 0x6b, 0xe8, 0x46, 0x7e, 0xb0, 0x97, 0x0c, 0x9b, 0x80, 0x15, 0x04, 0x6b, 0x58, 0xbc, 0xe5, - 0x5d, 0x97, 0x59, 0x9a, 0xe7, 0x4c, 0x3d, 0x1c, 0x16, 0xe5, 0x58, 0x61, 0xd0, 0x0f, 0xab, 0xbb, - 0x01, 0x13, 0x61, 0xee, 0x89, 0x13, 0x56, 0x7d, 0x58, 0x49, 0x02, 0x70, 0x8c, 0x43, 0x5f, 0x73, - 0x73, 0xd7, 0x1a, 0xed, 0x30, 0x22, 0xc1, 0x66, 0x98, 0x71, 0x3a, 0x3e, 0x07, 0x45, 0x22, 0x15, + 0xd6, 0x63, 0xaa, 0x13, 0x94, 0x89, 0xd9, 0xc7, 0x1c, 0x86, 0x5e, 0x86, 0xb1, 0x80, 0xb0, 0x55, + 0xb1, 0x4c, 0xb9, 0x39, 0xb6, 0xcc, 0x06, 0x63, 0xb6, 0x0f, 0x6b, 0x30, 0x6c, 0x60, 0xc6, 0x6f, + 0x83, 0xa1, 0x2e, 0x6f, 0x83, 0xa7, 0x60, 0x98, 0xfd, 0x50, 0x2b, 0x40, 0xcd, 0x46, 0x85, 0x17, + 0x63, 0x09, 0x4f, 0x2e, 0x98, 0x91, 0xfe, 0x16, 0x0c, 0x7d, 0x7d, 0x88, 0x45, 0xcd, 0xb4, 0xcc, + 0x23, 0xfc, 0x94, 0x13, 0x4b, 0x1e, 0x4b, 0x98, 0xfd, 0x41, 0x98, 0x28, 0x3b, 0xa4, 0xe5, 0x7b, + 0x2b, 0x5e, 0xa3, 0xed, 0xbb, 0x5e, 0x84, 0x66, 0x61, 0x80, 0x5d, 0x22, 0xfc, 0x08, 0x18, 0xa0, + 0x0d, 0xe1, 0x01, 0xfa, 0x20, 0xb0, 0xb7, 0xe1, 0x74, 0xd9, 0xbf, 0xe7, 0xdd, 0x73, 0x82, 0xc6, + 0x62, 0xb5, 0xa2, 0xbd, 0xaf, 0xd7, 0xe5, 0xfb, 0x8e, 0x1b, 0x6d, 0x65, 0x1e, 0xbd, 0x5a, 0x4d, + 0xce, 0xd6, 0xae, 0xba, 0x4d, 0x92, 0x23, 0x05, 0xf9, 0xab, 0x05, 0xa3, 0xa5, 0x18, 0x5f, 0x69, + 0xb5, 0xac, 0x5c, 0xad, 0xd6, 0x1b, 0x30, 0xb2, 0xe5, 0x92, 0x66, 0x03, 0x93, 0x2d, 0xb1, 0x12, + 0x9f, 0xcc, 0xb7, 0x43, 0x59, 0xa5, 0x98, 0x52, 0xea, 0xc5, 0x5f, 0x87, 0xab, 0xa2, 0x32, 0x56, + 0x64, 0xd0, 0x2e, 0x4c, 0xc9, 0x07, 0x83, 0x84, 0x8a, 0x75, 0xf9, 0x54, 0xb7, 0x57, 0x88, 0x49, + 0xfc, 0xd4, 0x83, 0x83, 0xf9, 0x29, 0x9c, 0x20, 0x83, 0x53, 0x84, 0xe9, 0x73, 0xb0, 0x45, 0x4f, + 0xe0, 0x01, 0x36, 0xfc, 0xec, 0x39, 0xc8, 0x5e, 0xb6, 0xac, 0xd4, 0xfe, 0x31, 0x0b, 0x1e, 0x4b, + 0x8d, 0x8c, 0x78, 0xe1, 0x1f, 0xf3, 0x2c, 0x24, 0x5f, 0xdc, 0x85, 0xde, 0x2f, 0x6e, 0xfb, 0xef, + 0x58, 0x70, 0x6a, 0xa5, 0xd5, 0x8e, 0xf6, 0xcb, 0xae, 0xa9, 0x82, 0xfa, 0x08, 0x0c, 0xb5, 0x48, + 0xc3, 0xed, 0xb4, 0xc4, 0xcc, 0xcd, 0xcb, 0x53, 0x6a, 0x8d, 0x95, 0x1e, 0x1e, 0xcc, 0x8f, 0xd7, + 0x22, 0x3f, 0x70, 0xb6, 0x09, 0x2f, 0xc0, 0x02, 0x9d, 0x9d, 0xf5, 0xee, 0xdb, 0xe4, 0xa6, 0xdb, + 0x72, 0xa5, 0x5d, 0x51, 0x57, 0x99, 0xdd, 0x82, 0x1c, 0xd0, 0x85, 0x37, 0x3a, 0x8e, 0x17, 0xb9, + 0xd1, 0xbe, 0xd0, 0x1e, 0x49, 0x22, 0x38, 0xa6, 0x67, 0x7f, 0xd5, 0x82, 0x49, 0xb9, 0xee, 0x17, + 0x1b, 0x8d, 0x80, 0x84, 0x21, 0x9a, 0x83, 0x82, 0xdb, 0x16, 0xbd, 0x04, 0xd1, 0xcb, 0x42, 0xa5, + 0x8a, 0x0b, 0x6e, 0x5b, 0xb2, 0x65, 0xec, 0x20, 0x2c, 0x9a, 0x8a, 0xb4, 0xeb, 0xa2, 0x1c, 0x2b, + 0x0c, 0x74, 0x05, 0x46, 0x3c, 0xbf, 0xc1, 0x6d, 0xbb, 0xf8, 0x95, 0xc6, 0x16, 0xd8, 0xba, 0x28, + 0xc3, 0x0a, 0x8a, 0xaa, 0x50, 0xe2, 0x66, 0x4f, 0xf1, 0xa2, 0xed, 0xcb, 0x78, 0x8a, 0x7d, 0xd9, + 0x86, 0xac, 0x89, 0x63, 0x22, 0xf6, 0xaf, 0x5a, 0x30, 0x26, 0xbf, 0xac, 0x4f, 0x9e, 0x93, 0x6e, + 0xad, 0x98, 0xdf, 0x8c, 0xb7, 0x16, 0xe5, 0x19, 0x19, 0xc4, 0x60, 0x15, 0x8b, 0x47, 0x62, 0x15, + 0x9f, 0x83, 0x51, 0xa7, 0xdd, 0xae, 0x9a, 0x7c, 0x26, 0x5b, 0x4a, 0x8b, 0x71, 0x31, 0xd6, 0x71, + 0xec, 0x1f, 0x2d, 0xc0, 0x84, 0xfc, 0x82, 0x5a, 0x67, 0x33, 0x24, 0x11, 0xda, 0x80, 0x92, 0xc3, + 0x67, 0x89, 0xc8, 0x45, 0x7e, 0x29, 0x5b, 0x8e, 0x60, 0x4c, 0x69, 0x7c, 0xe1, 0x2f, 0xca, 0xda, + 0x38, 0x26, 0x84, 0x9a, 0x30, 0xed, 0xf9, 0x11, 0x3b, 0xfc, 0x15, 0xbc, 0x9b, 0x6a, 0x27, 0x49, + 0xfd, 0xac, 0xa0, 0x3e, 0xbd, 0x9e, 0xa4, 0x82, 0xd3, 0x84, 0xd1, 0x8a, 0x94, 0xcd, 0x14, 0xf3, + 0x85, 0x01, 0xfa, 0xc4, 0x65, 0x8b, 0x66, 0xec, 0x5f, 0xb1, 0xa0, 0x24, 0xd1, 0x4e, 0x42, 0x8b, + 0xb7, 0x06, 0xc3, 0x21, 0x9b, 0x04, 0x39, 0x34, 0x76, 0xb7, 0x8e, 0xf3, 0xf9, 0x8a, 0xef, 0x34, + 0xfe, 0x3f, 0xc4, 0x92, 0x06, 0x13, 0xcd, 0xab, 0xee, 0xbf, 0x4b, 0x44, 0xf3, 0xaa, 0x3f, 0x39, + 0x97, 0xd2, 0x1f, 0xb1, 0x3e, 0x6b, 0xb2, 0x2e, 0xca, 0x7a, 0xb5, 0x03, 0xb2, 0xe5, 0xde, 0x4f, + 0xb2, 0x5e, 0x55, 0x56, 0x8a, 0x05, 0x14, 0xbd, 0x09, 0x63, 0x75, 0x29, 0x93, 0x8d, 0x77, 0xf8, + 0xe5, 0xae, 0xfa, 0x01, 0xa5, 0x4a, 0xe2, 0xb2, 0x90, 0x65, 0xad, 0x3e, 0x36, 0xa8, 0x99, 0x66, + 0x04, 0xc5, 0x5e, 0x66, 0x04, 0x31, 0xdd, 0x7c, 0xa5, 0xfa, 0x8f, 0x5b, 0x30, 0xc4, 0x65, 0x71, + 0xfd, 0x89, 0x42, 0x35, 0xcd, 0x5a, 0x3c, 0x76, 0x77, 0x68, 0xa1, 0xd0, 0x94, 0xa1, 0x35, 0x28, + 0xb1, 0x1f, 0x4c, 0x96, 0x58, 0xcc, 0xb7, 0xba, 0xe7, 0xad, 0xea, 0x1d, 0xbc, 0x23, 0xab, 0xe1, + 0x98, 0x82, 0xfd, 0xc3, 0x45, 0x7a, 0xba, 0xc5, 0xa8, 0xc6, 0xa5, 0x6f, 0x3d, 0xba, 0x4b, 0xbf, + 0xf0, 0xa8, 0x2e, 0xfd, 0x6d, 0x98, 0xac, 0x6b, 0x7a, 0xb8, 0x78, 0x26, 0xaf, 0x74, 0x5d, 0x24, + 0x9a, 0xca, 0x8e, 0x4b, 0x59, 0x96, 0x4d, 0x22, 0x38, 0x49, 0x15, 0x7d, 0x1b, 0x8c, 0xf1, 0x79, + 0x16, 0xad, 0x70, 0x4b, 0x8c, 0x0f, 0xe4, 0xaf, 0x17, 0xbd, 0x09, 0x2e, 0x95, 0xd3, 0xaa, 0x63, + 0x83, 0x98, 0xfd, 0xa7, 0x16, 0xa0, 0x95, 0xf6, 0x0e, 0x69, 0x91, 0xc0, 0x69, 0xc6, 0xe2, 0xf4, + 0xef, 0xb7, 0x60, 0x96, 0xa4, 0x8a, 0x97, 0xfd, 0x56, 0x4b, 0x3c, 0x5a, 0x72, 0xde, 0xd5, 0x2b, + 0x39, 0x75, 0x94, 0x5b, 0xc2, 0x6c, 0x1e, 0x06, 0xce, 0x6d, 0x0f, 0xad, 0xc1, 0x0c, 0xbf, 0x25, + 0x15, 0x40, 0xb3, 0xbd, 0x7e, 0x5c, 0x10, 0x9e, 0xd9, 0x48, 0xa3, 0xe0, 0xac, 0x7a, 0xf6, 0x77, + 0x8d, 0x41, 0x6e, 0x2f, 0xde, 0xd3, 0x23, 0xbc, 0xa7, 0x47, 0x78, 0x4f, 0x8f, 0xf0, 0x9e, 0x1e, + 0xe1, 0x3d, 0x3d, 0xc2, 0x37, 0xbd, 0x1e, 0xe1, 0x8f, 0x2d, 0x98, 0x49, 0x5f, 0x03, 0x27, 0xc1, + 0x98, 0x77, 0x60, 0x26, 0x7d, 0xd7, 0x75, 0xb5, 0xb3, 0x4b, 0xf7, 0x33, 0xbe, 0xf7, 0x32, 0xbe, + 0x01, 0x67, 0xd1, 0xb7, 0x7f, 0xc3, 0x82, 0xd3, 0x0a, 0xd9, 0x78, 0xe9, 0x7f, 0x1e, 0x66, 0xf8, + 0xf9, 0xb2, 0xdc, 0x74, 0xdc, 0xd6, 0x06, 0x69, 0xb5, 0x9b, 0x4e, 0x24, 0xcd, 0x0c, 0x9e, 0xcb, + 0xdc, 0xaa, 0x09, 0x13, 0x5d, 0xa3, 0xe2, 0xd2, 0x63, 0xb4, 0x5f, 0x19, 0x00, 0x9c, 0xd5, 0x8c, + 0x61, 0x94, 0x5a, 0xe8, 0x69, 0x26, 0xfc, 0x8b, 0x23, 0x30, 0xb8, 0xb2, 0x47, 0xbc, 0xe8, 0x04, + 0x26, 0xaa, 0x0e, 0x13, 0xae, 0xb7, 0xe7, 0x37, 0xf7, 0x48, 0x83, 0xc3, 0x8f, 0xf2, 0xd0, 0x3f, + 0x23, 0x48, 0x4f, 0x54, 0x0c, 0x12, 0x38, 0x41, 0xf2, 0x51, 0x08, 0xdb, 0xaf, 0xc1, 0x10, 0xbf, + 0xe3, 0x84, 0xa4, 0x3d, 0xf3, 0x4a, 0x63, 0x83, 0x28, 0x6e, 0xee, 0x58, 0x11, 0xc0, 0xef, 0x50, + 0x51, 0x1d, 0x7d, 0x0e, 0x26, 0xb6, 0xdc, 0x20, 0x8c, 0x36, 0xdc, 0x16, 0x09, 0x23, 0xa7, 0xd5, + 0x7e, 0x08, 0xe1, 0xba, 0x1a, 0x87, 0x55, 0x83, 0x12, 0x4e, 0x50, 0x46, 0xdb, 0x30, 0xde, 0x74, + 0xf4, 0xa6, 0x86, 0x8f, 0xdc, 0x94, 0xba, 0x3c, 0x6f, 0xea, 0x84, 0xb0, 0x49, 0x97, 0x9e, 0x36, + 0x75, 0x26, 0x1f, 0x1e, 0x61, 0x52, 0x13, 0x75, 0xda, 0x70, 0xc1, 0x30, 0x87, 0x51, 0x3e, 0x90, + 0xd9, 0x0f, 0x97, 0x4c, 0x3e, 0x50, 0xb3, 0x12, 0xfe, 0x2c, 0x94, 0x08, 0x1d, 0x42, 0x4a, 0x58, + 0xdc, 0xbf, 0x57, 0xfb, 0xeb, 0xeb, 0x9a, 0x5b, 0x0f, 0x7c, 0x53, 0xad, 0xb1, 0x22, 0x29, 0xe1, + 0x98, 0x28, 0x5a, 0x86, 0xa1, 0x90, 0x04, 0x2e, 0x09, 0xc5, 0x4d, 0xdc, 0x65, 0x1a, 0x19, 0x1a, + 0x77, 0xbd, 0xe1, 0xbf, 0xb1, 0xa8, 0x4a, 0x97, 0x97, 0xc3, 0x24, 0xbe, 0xec, 0xae, 0xd4, 0x96, + 0xd7, 0x22, 0x2b, 0xc5, 0x02, 0x8a, 0x5e, 0x87, 0xe1, 0x80, 0x34, 0x99, 0xde, 0x6c, 0xbc, 0xff, + 0x45, 0xce, 0xd5, 0x70, 0xbc, 0x1e, 0x96, 0x04, 0xd0, 0x0d, 0x40, 0x01, 0xa1, 0x7c, 0xa4, 0xeb, + 0x6d, 0x2b, 0xab, 0x5a, 0x71, 0x0f, 0xa9, 0x73, 0x0b, 0xc7, 0x18, 0xd2, 0x0b, 0x0a, 0x67, 0x54, + 0x43, 0xd7, 0x60, 0x5a, 0x95, 0x56, 0xbc, 0x30, 0x72, 0xe8, 0xf9, 0x3f, 0xc9, 0x68, 0x29, 0x31, + 0x0e, 0x4e, 0x22, 0xe0, 0x74, 0x1d, 0xfb, 0xcb, 0x16, 0xf0, 0x71, 0x3e, 0x01, 0xe1, 0xc5, 0x6b, + 0xa6, 0xf0, 0xe2, 0x6c, 0xee, 0xcc, 0xe5, 0x08, 0x2e, 0xbe, 0x6c, 0xc1, 0xa8, 0x36, 0xb3, 0xf1, + 0x9a, 0xb5, 0xba, 0xac, 0xd9, 0x0e, 0x4c, 0xd1, 0x95, 0x7e, 0x6b, 0x33, 0x24, 0xc1, 0x1e, 0x69, + 0xb0, 0x85, 0x59, 0x78, 0xb8, 0x85, 0xa9, 0x2c, 0xf8, 0x6e, 0x26, 0x08, 0xe2, 0x54, 0x13, 0xf6, + 0x67, 0x65, 0x57, 0x95, 0xc1, 0x63, 0x5d, 0xcd, 0x79, 0xc2, 0xe0, 0x51, 0xcd, 0x2a, 0x8e, 0x71, + 0xe8, 0x56, 0xdb, 0xf1, 0xc3, 0x28, 0x69, 0xf0, 0x78, 0xdd, 0x0f, 0x23, 0xcc, 0x20, 0xf6, 0x0b, + 0x00, 0x2b, 0xf7, 0x49, 0x9d, 0xaf, 0x58, 0xfd, 0x6d, 0x65, 0xe5, 0xbf, 0xad, 0xec, 0xdf, 0xb1, + 0x60, 0x62, 0x75, 0xd9, 0xb8, 0xe7, 0x16, 0x00, 0xf8, 0x83, 0xf0, 0xee, 0xdd, 0x75, 0x69, 0x2d, + 0xc0, 0x15, 0xbe, 0xaa, 0x14, 0x6b, 0x18, 0xe8, 0x2c, 0x14, 0x9b, 0x1d, 0x4f, 0x48, 0x57, 0x87, + 0x29, 0xf7, 0x70, 0xb3, 0xe3, 0x61, 0x5a, 0xa6, 0x79, 0x5c, 0x14, 0xfb, 0xf6, 0xb8, 0xe8, 0x19, + 0xf9, 0x00, 0xcd, 0xc3, 0xe0, 0xbd, 0x7b, 0x6e, 0x83, 0xfb, 0x97, 0x0a, 0x4b, 0x86, 0xbb, 0x77, + 0x2b, 0xe5, 0x10, 0xf3, 0x72, 0xfb, 0x4b, 0x45, 0x98, 0x5b, 0x6d, 0x92, 0xfb, 0xef, 0xd0, 0xc7, + 0xb6, 0x5f, 0x7f, 0x91, 0xa3, 0xc9, 0xa9, 0x8e, 0xea, 0x13, 0xd4, 0x7b, 0x3c, 0xb6, 0x60, 0x98, + 0xdb, 0xfb, 0x49, 0x8f, 0xdb, 0x57, 0xb3, 0x5a, 0xcf, 0x1f, 0x90, 0x05, 0x6e, 0x37, 0x28, 0x1c, + 0x06, 0xd5, 0x85, 0x29, 0x4a, 0xb1, 0x24, 0x3e, 0xf7, 0x0a, 0x8c, 0xe9, 0x98, 0x47, 0xf2, 0xce, + 0xfb, 0x7f, 0x8b, 0x30, 0x45, 0x7b, 0xf0, 0x48, 0x27, 0xe2, 0x76, 0x7a, 0x22, 0x8e, 0xdb, 0x43, + 0xab, 0xf7, 0x6c, 0xbc, 0x99, 0x9c, 0x8d, 0xe7, 0xf2, 0x66, 0xe3, 0xa4, 0xe7, 0xe0, 0x3b, 0x2d, + 0x98, 0x59, 0x6d, 0xfa, 0xf5, 0xdd, 0x84, 0x17, 0xd5, 0x4b, 0x30, 0x4a, 0x8f, 0xe3, 0xd0, 0x70, + 0xf0, 0x37, 0x42, 0x3e, 0x08, 0x10, 0xd6, 0xf1, 0xb4, 0x6a, 0xb7, 0x6f, 0x57, 0xca, 0x59, 0x91, + 0x22, 0x04, 0x08, 0xeb, 0x78, 0xf6, 0x6f, 0x59, 0x70, 0xfe, 0xda, 0xf2, 0x4a, 0xbc, 0x14, 0x53, + 0xc1, 0x2a, 0x2e, 0xc3, 0x50, 0xbb, 0xa1, 0x75, 0x25, 0x96, 0x3e, 0x97, 0x59, 0x2f, 0x04, 0xf4, + 0xdd, 0x12, 0x88, 0xe5, 0xa7, 0x2d, 0x98, 0xb9, 0xe6, 0x46, 0xf4, 0x76, 0x4d, 0x86, 0x4d, 0xa0, + 0xd7, 0x6b, 0xe8, 0x46, 0x7e, 0xb0, 0x9f, 0x0c, 0x9b, 0x80, 0x15, 0x04, 0x6b, 0x58, 0xbc, 0xe5, + 0x3d, 0x97, 0x59, 0x9a, 0x17, 0x4c, 0x3d, 0x1c, 0x16, 0xe5, 0x58, 0x61, 0xd0, 0x0f, 0x6b, 0xb8, + 0x01, 0x13, 0x61, 0xee, 0x8b, 0x13, 0x56, 0x7d, 0x58, 0x59, 0x02, 0x70, 0x8c, 0x43, 0x5f, 0x73, + 0xf3, 0xd7, 0x9a, 0x9d, 0x30, 0x22, 0xc1, 0x56, 0x98, 0x73, 0x3a, 0xbe, 0x00, 0x25, 0x22, 0x15, 0x06, 0xa2, 0xd7, 0x8a, 0x63, 0x54, 0x9a, 0x04, 0x1e, 0xbd, 0x41, 0xe1, 0xf5, 0xe1, 0x93, 0x79, - 0x38, 0xa7, 0xba, 0x15, 0x40, 0x44, 0x6f, 0x4b, 0x0f, 0x67, 0xc1, 0xfc, 0xe2, 0x97, 0x3b, 0xa0, - 0x38, 0xa5, 0x86, 0xfd, 0x63, 0x16, 0x9c, 0x56, 0x1f, 0xfc, 0xbe, 0xfb, 0x4c, 0xfb, 0xe7, 0x72, - 0x30, 0x76, 0x7d, 0x7d, 0xbd, 0x72, 0x8d, 0x44, 0xe2, 0xda, 0xee, 0x6d, 0x06, 0x80, 0x35, 0x6d, - 0x66, 0xb7, 0xc7, 0x5c, 0x3b, 0x72, 0x1b, 0xf3, 0x3c, 0x2a, 0xd2, 0x7c, 0xd9, 0x8b, 0x6e, 0x05, - 0xd5, 0x28, 0x70, 0xbd, 0xad, 0x54, 0xfd, 0xa7, 0x64, 0x2e, 0xf2, 0x59, 0xcc, 0x05, 0x7a, 0x0e, - 0x86, 0x58, 0x58, 0x26, 0x39, 0x09, 0x8f, 0xaa, 0xb7, 0x10, 0x2b, 0x3d, 0xd8, 0x9f, 0x2b, 0xde, - 0xc6, 0x65, 0xfe, 0x07, 0x0b, 0x54, 0x74, 0x1b, 0x46, 0xb6, 0xa3, 0xa8, 0x75, 0x9d, 0x38, 0x75, - 0xfa, 0x74, 0xe7, 0xc7, 0xe1, 0x85, 0xb4, 0xe3, 0x90, 0x0e, 0x02, 0x47, 0x8b, 0x4f, 0x90, 0xb8, - 0x2c, 0xc4, 0x3a, 0x1d, 0xbb, 0x0a, 0x10, 0xc3, 0x8e, 0x48, 0x91, 0x63, 0xff, 0x81, 0x05, 0xc3, - 0x3c, 0x42, 0x46, 0x80, 0x5e, 0x85, 0x01, 0x72, 0x9f, 0xd4, 0x04, 0xc7, 0x9b, 0xda, 0xe1, 0x98, + 0x34, 0xa7, 0xba, 0x55, 0x40, 0x44, 0x6f, 0x4b, 0x0f, 0x67, 0xc1, 0xfc, 0xe2, 0x57, 0x52, 0x50, + 0x9c, 0x51, 0xc3, 0xfe, 0x31, 0x0b, 0x4e, 0xab, 0x0f, 0x7e, 0xd7, 0x7d, 0xa6, 0xfd, 0x73, 0x05, + 0x18, 0xbf, 0xbe, 0xb1, 0x51, 0xbd, 0x46, 0x22, 0x71, 0x6d, 0xf7, 0x36, 0x03, 0xc0, 0x9a, 0x36, + 0xb3, 0xdb, 0x63, 0xae, 0x13, 0xb9, 0xcd, 0x05, 0x1e, 0x15, 0x69, 0xa1, 0xe2, 0x45, 0xb7, 0x82, + 0x5a, 0x14, 0xb8, 0xde, 0x76, 0xa6, 0xfe, 0x53, 0x32, 0x17, 0xc5, 0x3c, 0xe6, 0x02, 0xbd, 0x00, + 0x43, 0x2c, 0x2c, 0x93, 0x9c, 0x84, 0xc7, 0xd5, 0x5b, 0x88, 0x95, 0x1e, 0x1e, 0xcc, 0x97, 0x6e, + 0xe3, 0x0a, 0xff, 0x83, 0x05, 0x2a, 0xba, 0x0d, 0xa3, 0x3b, 0x51, 0xd4, 0xbe, 0x4e, 0x9c, 0x06, + 0x7d, 0xba, 0xf3, 0xe3, 0xf0, 0x42, 0xd6, 0x71, 0x48, 0x07, 0x81, 0xa3, 0xc5, 0x27, 0x48, 0x5c, + 0x16, 0x62, 0x9d, 0x8e, 0x5d, 0x03, 0x88, 0x61, 0xc7, 0xa4, 0xc8, 0xb1, 0xff, 0xd0, 0x82, 0x61, + 0x1e, 0x21, 0x23, 0x40, 0x1f, 0x85, 0x01, 0x72, 0x9f, 0xd4, 0x05, 0xc7, 0x9b, 0xd9, 0xe1, 0x98, 0xd3, 0xe2, 0x02, 0x69, 0xfa, 0x1f, 0xb3, 0x5a, 0xe8, 0x3a, 0x0c, 0xd3, 0xde, 0x5e, 0x53, 0xe1, - 0x42, 0x1e, 0xcb, 0xfa, 0x62, 0x35, 0xed, 0x9c, 0x39, 0x13, 0x45, 0x58, 0x56, 0x67, 0xda, 0xf3, - 0x5a, 0xab, 0x4a, 0x4f, 0xec, 0xa8, 0x1b, 0x63, 0xb1, 0xbe, 0x54, 0xe1, 0x48, 0x82, 0x1a, 0xd7, - 0x9e, 0xcb, 0x42, 0x1c, 0x13, 0xb1, 0xd7, 0xa1, 0x48, 0x27, 0x75, 0xa1, 0xe1, 0x3a, 0xdd, 0x0d, - 0x02, 0x9e, 0x84, 0xa2, 0x54, 0xf7, 0x87, 0xc2, 0x33, 0x9e, 0x51, 0x95, 0xd6, 0x00, 0x21, 0x8e, - 0xe1, 0xf6, 0x26, 0x9c, 0x62, 0xc6, 0x9b, 0x4e, 0xb4, 0x6d, 0xec, 0xb1, 0xde, 0x8b, 0xf9, 0x29, - 0xf1, 0x80, 0xe4, 0x33, 0x33, 0xa3, 0x39, 0x9f, 0x8e, 0x4a, 0x8a, 0xf1, 0x63, 0xd2, 0xfe, 0xda, - 0x00, 0x3c, 0x5a, 0xae, 0x66, 0x07, 0x4f, 0x79, 0x09, 0x46, 0x39, 0x5f, 0x4a, 0x97, 0xb6, 0xd3, - 0x10, 0xed, 0x2a, 0x49, 0xf4, 0xba, 0x06, 0xc3, 0x06, 0x26, 0x3a, 0x0f, 0x79, 0xf7, 0x1d, 0x2f, - 0xe9, 0x9a, 0x55, 0x7e, 0x63, 0x0d, 0xd3, 0x72, 0x0a, 0xa6, 0x2c, 0x2e, 0xbf, 0x3b, 0x14, 0x58, - 0xb1, 0xb9, 0xaf, 0xc1, 0xb8, 0x1b, 0xd6, 0x42, 0xb7, 0xec, 0xd1, 0x73, 0x46, 0x3b, 0xa9, 0x94, + 0x42, 0x9e, 0xc8, 0xfb, 0x62, 0x35, 0xed, 0x9c, 0x39, 0x13, 0x45, 0x58, 0x56, 0x67, 0xda, 0xf3, + 0x7a, 0xbb, 0x46, 0x4f, 0xec, 0xa8, 0x1b, 0x63, 0xb1, 0xb1, 0x5c, 0xe5, 0x48, 0x82, 0x1a, 0xd7, + 0x9e, 0xcb, 0x42, 0x1c, 0x13, 0xb1, 0x37, 0xa0, 0x44, 0x27, 0x75, 0xb1, 0xe9, 0x3a, 0xdd, 0x0d, + 0x02, 0x9e, 0x86, 0x92, 0x54, 0xf7, 0x87, 0xc2, 0x33, 0x9e, 0x51, 0x95, 0xd6, 0x00, 0x21, 0x8e, + 0xe1, 0xf6, 0x16, 0x9c, 0x62, 0xc6, 0x9b, 0x4e, 0xb4, 0x63, 0xec, 0xb1, 0xde, 0x8b, 0xf9, 0x19, + 0xf1, 0x80, 0xe4, 0x33, 0x33, 0xab, 0x39, 0x9f, 0x8e, 0x49, 0x8a, 0xf1, 0x63, 0xd2, 0xfe, 0xda, + 0x00, 0x3c, 0x5e, 0xa9, 0xe5, 0x07, 0x4f, 0x79, 0x19, 0xc6, 0x38, 0x5f, 0x4a, 0x97, 0xb6, 0xd3, + 0x14, 0xed, 0x2a, 0x49, 0xf4, 0x86, 0x06, 0xc3, 0x06, 0x26, 0x3a, 0x0f, 0x45, 0xf7, 0x2d, 0x2f, + 0xe9, 0x9a, 0x55, 0x79, 0x63, 0x1d, 0xd3, 0x72, 0x0a, 0xa6, 0x2c, 0x2e, 0xbf, 0x3b, 0x14, 0x58, + 0xb1, 0xb9, 0xaf, 0xc1, 0x84, 0x1b, 0xd6, 0x43, 0xb7, 0xe2, 0xd1, 0x73, 0x46, 0x3b, 0xa9, 0x94, 0x70, 0x83, 0x76, 0x5a, 0x41, 0x71, 0x02, 0x5b, 0xbb, 0xc8, 0x06, 0xfb, 0x66, 0x93, 0x7b, 0xba, - 0x8a, 0xd3, 0x17, 0x40, 0x8b, 0x7d, 0x5d, 0xc8, 0x54, 0x0a, 0xe2, 0x05, 0xc0, 0x3f, 0x38, 0xc4, - 0x12, 0x46, 0x5f, 0x8e, 0xb5, 0x6d, 0xa7, 0xb5, 0xd0, 0x8e, 0xb6, 0x4b, 0x6e, 0x58, 0xf3, 0x77, - 0x49, 0xb0, 0xc7, 0x1e, 0xfd, 0x85, 0xf8, 0xe5, 0xa8, 0x00, 0x4b, 0xd7, 0x17, 0x2a, 0x14, 0x13, - 0x77, 0xd6, 0x41, 0x0b, 0x30, 0x21, 0x0b, 0xab, 0x24, 0x64, 0x57, 0xd8, 0x08, 0x23, 0xa3, 0x9c, - 0xa5, 0x44, 0xb1, 0x22, 0x92, 0xc4, 0x37, 0x39, 0x69, 0x38, 0x0a, 0x4e, 0xfa, 0x45, 0x18, 0x73, - 0x3d, 0x37, 0x72, 0x9d, 0xc8, 0xe7, 0xfa, 0x30, 0xfe, 0xbe, 0x67, 0x82, 0xfe, 0xb2, 0x0e, 0xc0, - 0x26, 0x9e, 0xfd, 0x5f, 0x06, 0x60, 0x8a, 0x4d, 0xdb, 0x07, 0x2b, 0xec, 0x9b, 0x69, 0x85, 0xdd, - 0xee, 0x5c, 0x61, 0x47, 0xf1, 0x44, 0x78, 0xe8, 0x65, 0xf6, 0x36, 0x14, 0x95, 0x7f, 0x98, 0x74, - 0x10, 0xb5, 0x32, 0x1c, 0x44, 0x7b, 0x73, 0x1f, 0xd2, 0xc4, 0x2e, 0x9f, 0x6a, 0x62, 0xf7, 0xd7, - 0x2d, 0x88, 0x15, 0x3c, 0xe8, 0x3a, 0x14, 0x5b, 0x3e, 0xb3, 0x1c, 0x0d, 0xa4, 0x39, 0xf6, 0xa3, - 0xa9, 0x17, 0x15, 0xbf, 0x14, 0xf9, 0xc7, 0x57, 0x64, 0x0d, 0x1c, 0x57, 0x46, 0x8b, 0x30, 0xdc, - 0x0a, 0x48, 0x35, 0x62, 0x31, 0x54, 0x7a, 0xd2, 0xe1, 0x6b, 0x84, 0xe3, 0x63, 0x59, 0xd1, 0xfe, - 0x79, 0x0b, 0x80, 0x5b, 0xb1, 0x39, 0xde, 0x16, 0x39, 0x01, 0xa9, 0x75, 0x09, 0x06, 0xc2, 0x16, - 0xa9, 0x75, 0xb3, 0xe9, 0x8d, 0xfb, 0x53, 0x6d, 0x91, 0x5a, 0x3c, 0xe0, 0xf4, 0x1f, 0x66, 0xb5, - 0xed, 0xef, 0x06, 0x18, 0x8f, 0xd1, 0xca, 0x11, 0x69, 0xa2, 0xa7, 0x8d, 0x98, 0x0a, 0x67, 0x13, - 0x31, 0x15, 0x8a, 0x0c, 0x5b, 0x13, 0x90, 0xbe, 0x0d, 0xf9, 0xa6, 0x73, 0x5f, 0x48, 0xc0, 0x9e, - 0xec, 0xde, 0x0d, 0x4a, 0x7f, 0x7e, 0xd5, 0xb9, 0xcf, 0x1f, 0x89, 0x4f, 0xca, 0x05, 0xb2, 0xea, - 0xdc, 0x3f, 0xe0, 0x96, 0xbb, 0xec, 0x90, 0xba, 0xe9, 0x86, 0xd1, 0x17, 0xfe, 0x73, 0xfc, 0x9f, - 0x2d, 0x3b, 0xda, 0x08, 0x6b, 0xcb, 0xf5, 0x84, 0x81, 0x56, 0x5f, 0x6d, 0xb9, 0x5e, 0xb2, 0x2d, - 0xd7, 0xeb, 0xa3, 0x2d, 0xd7, 0x43, 0xef, 0xc2, 0xb0, 0xb0, 0x9f, 0x14, 0x31, 0x8c, 0xae, 0xf6, - 0xd1, 0x9e, 0x30, 0xbf, 0xe4, 0x6d, 0x5e, 0x95, 0x8f, 0x60, 0x51, 0xda, 0xb3, 0x5d, 0xd9, 0x20, - 0xfa, 0x2b, 0x16, 0x8c, 0x8b, 0xdf, 0x98, 0xbc, 0xd3, 0x26, 0x61, 0x24, 0x78, 0xcf, 0x8f, 0xf7, - 0xdf, 0x07, 0x51, 0x91, 0x77, 0xe5, 0xe3, 0xf2, 0x98, 0x35, 0x81, 0x3d, 0x7b, 0x94, 0xe8, 0x05, - 0xfa, 0x7b, 0x16, 0x9c, 0x6a, 0x3a, 0xf7, 0x79, 0x8b, 0xbc, 0x0c, 0x3b, 0x91, 0xeb, 0x0b, 0x3b, - 0x84, 0x57, 0xfb, 0x9b, 0xfe, 0x8e, 0xea, 0xbc, 0x93, 0x52, 0x59, 0x7a, 0x2a, 0x0d, 0xa5, 0x67, - 0x57, 0x53, 0xfb, 0x35, 0xbb, 0x09, 0x05, 0xb9, 0xde, 0x52, 0x44, 0x0d, 0x25, 0x9d, 0xb1, 0x3e, - 0xb4, 0xf9, 0xaa, 0x1e, 0xab, 0x80, 0xb6, 0x23, 0xd6, 0xda, 0xb1, 0xb6, 0xf3, 0x36, 0x8c, 0xea, - 0x6b, 0xec, 0x58, 0xdb, 0x7a, 0x07, 0xa6, 0x53, 0xd6, 0xd2, 0xb1, 0x36, 0x79, 0x0f, 0xce, 0x66, - 0xae, 0x8f, 0xe3, 0x6c, 0xd8, 0xfe, 0x39, 0x4b, 0x3f, 0x07, 0x4f, 0x40, 0x75, 0xb0, 0x64, 0xaa, - 0x0e, 0x2e, 0x74, 0xdf, 0x39, 0x19, 0xfa, 0x83, 0xb7, 0xf4, 0x4e, 0xd3, 0x53, 0x1d, 0xbd, 0x0e, - 0x43, 0x0d, 0x5a, 0x22, 0xad, 0x70, 0xed, 0xde, 0x3b, 0x32, 0xe6, 0xa5, 0x58, 0x79, 0x88, 0x05, - 0x05, 0xfb, 0x97, 0x2c, 0x18, 0x38, 0x81, 0x91, 0xc0, 0xe6, 0x48, 0x3c, 0x9d, 0x49, 0x5a, 0x84, - 0x57, 0x9e, 0xc7, 0xce, 0xbd, 0xe5, 0xfb, 0x11, 0xf1, 0x42, 0xf6, 0x54, 0x4c, 0x1d, 0x98, 0xff, - 0x07, 0xa6, 0x6f, 0xfa, 0x4e, 0x7d, 0xd1, 0x69, 0x38, 0x5e, 0x8d, 0x04, 0x65, 0x6f, 0xeb, 0x50, - 0x16, 0xe4, 0xb9, 0x5e, 0x16, 0xe4, 0xf6, 0x36, 0x20, 0xbd, 0x01, 0xe1, 0x8a, 0x83, 0x61, 0xd8, - 0xe5, 0x4d, 0x89, 0xe1, 0x7f, 0x3c, 0x9d, 0x35, 0xeb, 0xe8, 0x99, 0xe6, 0x64, 0xc2, 0x0b, 0xb0, - 0x24, 0x64, 0xbf, 0x04, 0xa9, 0xfe, 0xfc, 0xbd, 0xc5, 0x06, 0xf6, 0xa7, 0x61, 0x8a, 0xd5, 0x3c, - 0xe4, 0x93, 0xd6, 0x4e, 0x48, 0x25, 0x53, 0x22, 0xfd, 0xd9, 0x5f, 0xb4, 0x60, 0x62, 0x2d, 0x11, - 0x00, 0xed, 0x32, 0xd3, 0x63, 0xa6, 0x08, 0xc3, 0xab, 0xac, 0x14, 0x0b, 0xe8, 0x91, 0xcb, 0xa0, - 0xfe, 0xc2, 0x82, 0x38, 0xc4, 0xc6, 0x09, 0x30, 0x5e, 0x4b, 0x06, 0xe3, 0x95, 0x2a, 0x1b, 0x51, - 0xdd, 0xc9, 0xe2, 0xbb, 0xd0, 0x0d, 0x15, 0x7c, 0xaa, 0x8b, 0x58, 0x24, 0x26, 0xc3, 0x43, 0x15, - 0x8d, 0x9b, 0x11, 0xaa, 0x64, 0x38, 0x2a, 0xfb, 0x3f, 0xe6, 0x00, 0x29, 0xdc, 0xbe, 0x83, 0x63, - 0x75, 0xd6, 0x38, 0x9a, 0xe0, 0x58, 0xbb, 0x80, 0x98, 0x26, 0x3e, 0x70, 0xbc, 0x90, 0x93, 0x75, - 0x85, 0xd4, 0xed, 0x70, 0x6a, 0xfe, 0x59, 0xd1, 0x24, 0xba, 0xd9, 0x41, 0x0d, 0xa7, 0xb4, 0xa0, - 0x59, 0x58, 0x0c, 0xf6, 0x6b, 0x61, 0x31, 0xd4, 0xc3, 0xdd, 0xee, 0x67, 0x2d, 0x18, 0x53, 0xc3, - 0xf4, 0x3e, 0x31, 0x86, 0x57, 0xfd, 0xc9, 0x38, 0xfa, 0x2a, 0x5a, 0x97, 0xd9, 0x95, 0xf0, 0xad, - 0xcc, 0x6d, 0xd2, 0x69, 0xb8, 0xef, 0x12, 0x15, 0x9a, 0x70, 0x4e, 0xb8, 0x41, 0x8a, 0xd2, 0x83, - 0xfd, 0xb9, 0x31, 0xf5, 0x8f, 0x87, 0x42, 0x8e, 0xab, 0xd8, 0x3f, 0x49, 0x37, 0xbb, 0xb9, 0x14, - 0xd1, 0x0b, 0x30, 0xd8, 0xda, 0x76, 0x42, 0x92, 0x70, 0x1a, 0x1a, 0xac, 0xd0, 0xc2, 0x83, 0xfd, - 0xb9, 0x71, 0x55, 0x81, 0x95, 0x60, 0x8e, 0xdd, 0x7f, 0xc8, 0xb1, 0xce, 0xc5, 0xd9, 0x33, 0xe4, - 0xd8, 0x9f, 0x58, 0x30, 0xb0, 0xe6, 0xd7, 0x4f, 0xe2, 0x08, 0x78, 0xcd, 0x38, 0x02, 0xce, 0x65, - 0x45, 0xa9, 0xcf, 0xdc, 0xfd, 0x2b, 0x89, 0xdd, 0x7f, 0x21, 0x93, 0x42, 0xf7, 0x8d, 0xdf, 0x84, - 0x11, 0x16, 0xfb, 0x5e, 0x38, 0x48, 0x3d, 0x67, 0x6c, 0xf8, 0xb9, 0xc4, 0x86, 0x9f, 0xd0, 0x50, - 0xb5, 0x9d, 0xfe, 0x04, 0x0c, 0x0b, 0x8f, 0x9b, 0xa4, 0xf7, 0xa9, 0xc0, 0xc5, 0x12, 0x6e, 0xff, - 0x78, 0x1e, 0x8c, 0x58, 0xfb, 0xe8, 0x57, 0x2c, 0x98, 0x0f, 0xb8, 0x25, 0x6e, 0xbd, 0xd4, 0x0e, - 0x5c, 0x6f, 0xab, 0x5a, 0xdb, 0x26, 0xf5, 0x76, 0xc3, 0xf5, 0xb6, 0xca, 0x5b, 0x9e, 0xaf, 0x8a, - 0x97, 0xef, 0x93, 0x5a, 0x9b, 0xa9, 0xaf, 0x7a, 0x04, 0xf6, 0x57, 0x16, 0xed, 0xcf, 0x3e, 0xd8, - 0x9f, 0x9b, 0xc7, 0x87, 0xa2, 0x8d, 0x0f, 0xd9, 0x17, 0xf4, 0x9b, 0x16, 0x5c, 0xe5, 0x21, 0xe8, - 0xfb, 0xef, 0x7f, 0x97, 0x77, 0x6e, 0x45, 0x92, 0x8a, 0x89, 0xac, 0x93, 0xa0, 0xb9, 0xf8, 0xa2, - 0x18, 0xd0, 0xab, 0x95, 0xc3, 0xb5, 0x85, 0x0f, 0xdb, 0x39, 0xfb, 0x9f, 0xe6, 0x61, 0x4c, 0x84, - 0xa6, 0x12, 0x77, 0xc0, 0x0b, 0xc6, 0x92, 0x78, 0x2c, 0xb1, 0x24, 0xa6, 0x0c, 0xe4, 0xa3, 0x39, - 0xfe, 0x43, 0x98, 0xa2, 0x87, 0xf3, 0x75, 0xe2, 0x04, 0xd1, 0x06, 0x71, 0xb8, 0xe1, 0x54, 0xfe, - 0xd0, 0xa7, 0xbf, 0x12, 0xac, 0xdd, 0x4c, 0x12, 0xc3, 0x9d, 0xf4, 0xbf, 0x99, 0xee, 0x1c, 0x0f, - 0x26, 0x3b, 0xa2, 0x8b, 0xbd, 0x09, 0x45, 0xe5, 0x2e, 0x22, 0x0e, 0x9d, 0xee, 0x41, 0xfa, 0x92, - 0x14, 0xb8, 0xf0, 0x2b, 0x76, 0x55, 0x8a, 0xc9, 0xd9, 0x7f, 0x3f, 0x67, 0x34, 0xc8, 0x27, 0x71, - 0x0d, 0x0a, 0x4e, 0x18, 0xba, 0x5b, 0x1e, 0xa9, 0x8b, 0x1d, 0xfb, 0xe1, 0xac, 0x1d, 0x6b, 0x34, - 0xc3, 0x5c, 0x76, 0x16, 0x44, 0x4d, 0xac, 0x68, 0xa0, 0xeb, 0xdc, 0x3c, 0x6d, 0x57, 0xbe, 0xd4, - 0xfa, 0xa3, 0x06, 0xd2, 0x80, 0x6d, 0x97, 0x60, 0x51, 0x1f, 0x7d, 0x86, 0xdb, 0x0f, 0xde, 0xf0, - 0xfc, 0x7b, 0xde, 0x35, 0xdf, 0x97, 0xe1, 0x1f, 0xfa, 0x23, 0x38, 0x25, 0xad, 0x06, 0x55, 0x75, - 0x6c, 0x52, 0xeb, 0x2f, 0x5c, 0xe7, 0xe7, 0x61, 0x9a, 0x92, 0x36, 0xbd, 0xb3, 0x43, 0x44, 0x60, - 0x42, 0xc4, 0x3d, 0x93, 0x65, 0x62, 0xec, 0x52, 0x1f, 0x61, 0x66, 0xed, 0x58, 0x02, 0x7c, 0xc3, - 0x24, 0x81, 0x93, 0x34, 0xed, 0x9f, 0xb2, 0x80, 0x79, 0xaa, 0x9e, 0x00, 0x3f, 0xf2, 0x09, 0x93, - 0x1f, 0x99, 0xc9, 0x1a, 0xe4, 0x0c, 0x56, 0xe4, 0x79, 0xbe, 0xb2, 0x2a, 0x81, 0x7f, 0x7f, 0x4f, - 0x18, 0x7d, 0xf4, 0x7e, 0x7f, 0xd8, 0xff, 0xdb, 0xe2, 0x87, 0x98, 0x72, 0xe6, 0x40, 0xdf, 0x0e, - 0x85, 0x9a, 0xd3, 0x72, 0x6a, 0x3c, 0x31, 0x4c, 0xa6, 0x2c, 0xce, 0xa8, 0x34, 0xbf, 0x24, 0x6a, - 0x70, 0xd9, 0x92, 0x8c, 0x9f, 0x57, 0x90, 0xc5, 0x3d, 0xe5, 0x49, 0xaa, 0xc9, 0xd9, 0x1d, 0x18, - 0x33, 0x88, 0x1d, 0xab, 0x20, 0xe2, 0xdb, 0xf9, 0x15, 0xab, 0xe2, 0x3d, 0x36, 0x61, 0xca, 0xd3, - 0xfe, 0xd3, 0x0b, 0x45, 0x3e, 0x2e, 0x3f, 0xdc, 0xeb, 0x12, 0x65, 0xb7, 0x8f, 0xe6, 0x04, 0x9b, - 0x20, 0x83, 0x3b, 0x29, 0xdb, 0x3f, 0x61, 0xc1, 0x23, 0x3a, 0xa2, 0xe6, 0x67, 0xd3, 0x4b, 0xba, - 0x5f, 0x82, 0x82, 0xdf, 0x22, 0x81, 0x13, 0xf9, 0x81, 0xb8, 0x35, 0xae, 0xc8, 0x41, 0xbf, 0x25, - 0xca, 0x0f, 0x44, 0x58, 0x75, 0x49, 0x5d, 0x96, 0x63, 0x55, 0x93, 0xbe, 0x3e, 0xd9, 0x60, 0x84, - 0xc2, 0xa3, 0x8a, 0x9d, 0x01, 0x4c, 0xd1, 0x1d, 0x62, 0x01, 0xb1, 0xbf, 0x66, 0xf1, 0x85, 0xa5, - 0x77, 0x1d, 0xbd, 0x03, 0x93, 0x4d, 0x27, 0xaa, 0x6d, 0x2f, 0xdf, 0x6f, 0x05, 0x5c, 0x57, 0x22, - 0xc7, 0xe9, 0xc9, 0x5e, 0xe3, 0xa4, 0x7d, 0x64, 0x6c, 0x12, 0xb9, 0x9a, 0x20, 0x86, 0x3b, 0xc8, - 0xa3, 0x0d, 0x18, 0x61, 0x65, 0xcc, 0x59, 0x30, 0xec, 0xc6, 0x1a, 0x64, 0xb5, 0xa6, 0x6c, 0x05, - 0x56, 0x63, 0x3a, 0x58, 0x27, 0x6a, 0xff, 0x4c, 0x9e, 0xef, 0x76, 0xc6, 0xca, 0x3f, 0x01, 0xc3, - 0x2d, 0xbf, 0xbe, 0x54, 0x2e, 0x61, 0x31, 0x0b, 0xea, 0x1a, 0xa9, 0xf0, 0x62, 0x2c, 0xe1, 0xe8, - 0x0a, 0x14, 0xc4, 0x4f, 0xa9, 0xdb, 0x62, 0x67, 0xb3, 0xc0, 0x0b, 0xb1, 0x82, 0xa2, 0x67, 0x01, - 0x5a, 0x81, 0xbf, 0xeb, 0xd6, 0x59, 0x10, 0x8b, 0xbc, 0x69, 0xe6, 0x53, 0x51, 0x10, 0xac, 0x61, - 0xa1, 0x57, 0x60, 0xac, 0xed, 0x85, 0x9c, 0x1d, 0xd1, 0x42, 0xd6, 0x2a, 0x03, 0x94, 0xdb, 0x3a, - 0x10, 0x9b, 0xb8, 0x68, 0x01, 0x86, 0x22, 0x87, 0x99, 0xad, 0x0c, 0x66, 0x9b, 0xcd, 0xae, 0x53, - 0x0c, 0x3d, 0x07, 0x09, 0xad, 0x80, 0x45, 0x45, 0xf4, 0xa6, 0xf4, 0xdb, 0xe5, 0x07, 0xbb, 0xb0, - 0x57, 0xef, 0xef, 0x12, 0xd0, 0xbc, 0x76, 0x85, 0x1d, 0xbc, 0x41, 0x0b, 0xbd, 0x0c, 0x40, 0xee, - 0x47, 0x24, 0xf0, 0x9c, 0x86, 0xb2, 0x0a, 0x53, 0x7c, 0x41, 0xc9, 0x5f, 0xf3, 0xa3, 0xdb, 0x21, - 0x59, 0x56, 0x18, 0x58, 0xc3, 0xb6, 0x7f, 0xb3, 0x08, 0x10, 0xf3, 0xed, 0xe8, 0xdd, 0x8e, 0x83, - 0xeb, 0xa9, 0xee, 0x9c, 0xfe, 0xd1, 0x9d, 0x5a, 0xe8, 0x7b, 0x2c, 0x18, 0x71, 0x1a, 0x0d, 0xbf, - 0xe6, 0xf0, 0xa0, 0xc2, 0xb9, 0xee, 0x07, 0xa7, 0x68, 0x7f, 0x21, 0xae, 0xc1, 0xbb, 0xf0, 0x9c, - 0x5c, 0xa1, 0x1a, 0xa4, 0x67, 0x2f, 0xf4, 0x86, 0xd1, 0xc7, 0xe4, 0x53, 0x31, 0x6f, 0x0c, 0xa5, - 0x7a, 0x2a, 0x16, 0xd9, 0x1d, 0xa1, 0xbf, 0x12, 0x6f, 0x1b, 0xaf, 0xc4, 0x81, 0x6c, 0xc7, 0x44, - 0x83, 0x7d, 0xed, 0xf5, 0x40, 0x44, 0x15, 0x3d, 0x48, 0xc1, 0x60, 0xb6, 0x17, 0xa0, 0xf6, 0x4e, - 0xea, 0x11, 0xa0, 0xe0, 0x6d, 0x98, 0xa8, 0x9b, 0x4c, 0x80, 0x58, 0x89, 0x8f, 0x67, 0xd1, 0x4d, - 0xf0, 0x0c, 0xf1, 0xb5, 0x9f, 0x00, 0xe0, 0x24, 0x61, 0x54, 0xe1, 0x31, 0x2b, 0xca, 0xde, 0xa6, - 0x2f, 0x7c, 0x26, 0xec, 0xcc, 0xb9, 0xdc, 0x0b, 0x23, 0xd2, 0xa4, 0x98, 0xf1, 0xed, 0xbe, 0x26, - 0xea, 0x62, 0x45, 0x05, 0xbd, 0x0e, 0x43, 0xcc, 0x0d, 0x2c, 0x9c, 0x29, 0x64, 0xcb, 0x8a, 0xcd, - 0x20, 0x6c, 0xf1, 0x86, 0x64, 0x7f, 0x43, 0x2c, 0x28, 0xa0, 0xeb, 0xd2, 0xc9, 0x32, 0x2c, 0x7b, - 0xb7, 0x43, 0xc2, 0x9c, 0x2c, 0x8b, 0x8b, 0x1f, 0x8e, 0xfd, 0x27, 0x79, 0x79, 0x6a, 0xa6, 0x32, - 0xa3, 0x26, 0xe5, 0xa2, 0xc4, 0x7f, 0x99, 0x00, 0x6d, 0x06, 0xb2, 0xbb, 0x67, 0x26, 0x49, 0x8b, - 0x87, 0xf3, 0x8e, 0x49, 0x02, 0x27, 0x69, 0x52, 0x8e, 0x94, 0xef, 0x7a, 0xe1, 0x75, 0xd1, 0xeb, - 0xec, 0xe0, 0x0f, 0x71, 0x76, 0x1b, 0xf1, 0x12, 0x2c, 0xea, 0x9f, 0x28, 0x7b, 0x30, 0xeb, 0xc1, - 0x64, 0x72, 0x8b, 0x1e, 0x2b, 0x3b, 0xf2, 0x07, 0x03, 0x30, 0x6e, 0x2e, 0x29, 0x74, 0x15, 0x8a, - 0x82, 0x88, 0x4a, 0x5a, 0xa0, 0x76, 0xc9, 0xaa, 0x04, 0xe0, 0x18, 0x87, 0xe5, 0xaa, 0x60, 0xd5, - 0x35, 0x33, 0xdb, 0x38, 0x57, 0x85, 0x82, 0x60, 0x0d, 0x8b, 0x3e, 0xac, 0x36, 0x7c, 0x3f, 0x52, - 0x17, 0x92, 0x5a, 0x77, 0x8b, 0xac, 0x14, 0x0b, 0x28, 0xbd, 0x88, 0x76, 0x48, 0xe0, 0x91, 0x86, - 0x19, 0xde, 0x58, 0x5d, 0x44, 0x37, 0x74, 0x20, 0x36, 0x71, 0xe9, 0x75, 0xea, 0x87, 0x6c, 0x21, - 0x8b, 0xe7, 0x5b, 0x6c, 0xb6, 0x5c, 0xe5, 0x7e, 0xde, 0x12, 0x8e, 0x3e, 0x0d, 0x8f, 0xa8, 0x10, - 0x4e, 0x98, 0xeb, 0x21, 0x64, 0x8b, 0x43, 0x86, 0xb4, 0xe5, 0x91, 0xa5, 0x74, 0x34, 0x9c, 0x55, - 0x1f, 0xbd, 0x06, 0xe3, 0x82, 0xc5, 0x97, 0x14, 0x87, 0x4d, 0xd3, 0x98, 0x1b, 0x06, 0x14, 0x27, - 0xb0, 0x65, 0x80, 0x66, 0xc6, 0x65, 0x4b, 0x0a, 0x85, 0xce, 0x00, 0xcd, 0x3a, 0x1c, 0x77, 0xd4, - 0x40, 0x0b, 0x30, 0xc1, 0x79, 0x30, 0xd7, 0xdb, 0xe2, 0x73, 0x22, 0x9c, 0xa2, 0xd4, 0x96, 0xba, - 0x65, 0x82, 0x71, 0x12, 0x1f, 0xbd, 0x04, 0xa3, 0x4e, 0x50, 0xdb, 0x76, 0x23, 0x52, 0x8b, 0xda, - 0x01, 0xf7, 0x96, 0xd2, 0x6c, 0x8b, 0x16, 0x34, 0x18, 0x36, 0x30, 0xed, 0x77, 0x61, 0x3a, 0x25, - 0x00, 0x04, 0x5d, 0x38, 0x4e, 0xcb, 0x95, 0xdf, 0x94, 0x30, 0x40, 0x5e, 0xa8, 0x94, 0xe5, 0xd7, - 0x68, 0x58, 0x74, 0x75, 0xb2, 0x40, 0x11, 0x5a, 0xbe, 0x43, 0xb5, 0x3a, 0x57, 0x24, 0x00, 0xc7, - 0x38, 0xf6, 0xff, 0xc8, 0xc1, 0x44, 0x8a, 0x6e, 0x85, 0xe5, 0xdc, 0x4b, 0x3c, 0x52, 0xe2, 0x14, - 0x7b, 0x66, 0xbc, 0xef, 0xdc, 0x21, 0xe2, 0x7d, 0xe7, 0x7b, 0xc5, 0xfb, 0x1e, 0x78, 0x2f, 0xf1, - 0xbe, 0xcd, 0x11, 0x1b, 0xec, 0x6b, 0xc4, 0x52, 0x62, 0x84, 0x0f, 0x1d, 0x32, 0x46, 0xb8, 0x31, - 0xe8, 0xc3, 0x7d, 0x0c, 0xfa, 0x0f, 0xe7, 0x60, 0x32, 0x69, 0x03, 0x79, 0x02, 0x72, 0xdb, 0xd7, - 0x0d, 0xb9, 0xed, 0x95, 0x7e, 0x5c, 0x5e, 0x33, 0x65, 0xb8, 0x38, 0x21, 0xc3, 0xfd, 0x68, 0x5f, - 0xd4, 0xba, 0xcb, 0x73, 0xff, 0x66, 0x0e, 0x4e, 0xa7, 0xfa, 0xdc, 0x9e, 0xc0, 0xd8, 0xdc, 0x32, - 0xc6, 0xe6, 0xe9, 0xbe, 0xdd, 0x81, 0x33, 0x07, 0xe8, 0x6e, 0x62, 0x80, 0xae, 0xf6, 0x4f, 0xb2, - 0xfb, 0x28, 0x7d, 0x35, 0x0f, 0x17, 0x52, 0xeb, 0xc5, 0x62, 0xcf, 0x15, 0x43, 0xec, 0xf9, 0x6c, - 0x42, 0xec, 0x69, 0x77, 0xaf, 0x7d, 0x34, 0x72, 0x50, 0xe1, 0xe8, 0xca, 0xa2, 0x19, 0x3c, 0xa4, - 0x0c, 0xd4, 0x70, 0x74, 0x55, 0x84, 0xb0, 0x49, 0xf7, 0x9b, 0x49, 0xf6, 0xf9, 0xaf, 0x2d, 0x38, - 0x9b, 0x3a, 0x37, 0x27, 0x20, 0xeb, 0x5a, 0x33, 0x65, 0x5d, 0x4f, 0xf4, 0xbd, 0x5a, 0x33, 0x84, - 0x5f, 0xbf, 0x3e, 0x90, 0xf1, 0x2d, 0xec, 0x25, 0x7f, 0x0b, 0x46, 0x9c, 0x5a, 0x8d, 0x84, 0xe1, - 0xaa, 0x5f, 0x57, 0x21, 0x8d, 0x9f, 0x66, 0xef, 0xac, 0xb8, 0xf8, 0x60, 0x7f, 0x6e, 0x36, 0x49, - 0x22, 0x06, 0x63, 0x9d, 0x02, 0xfa, 0x0c, 0x14, 0x42, 0x71, 0x6f, 0x8a, 0xb9, 0x7f, 0xae, 0xcf, - 0xc1, 0x71, 0x36, 0x48, 0xc3, 0x8c, 0xb9, 0xa4, 0x24, 0x15, 0x8a, 0xa4, 0x19, 0x9f, 0x25, 0x77, - 0xa4, 0xf1, 0x59, 0x9e, 0x05, 0xd8, 0x55, 0x8f, 0x81, 0xa4, 0xfc, 0x41, 0x7b, 0x26, 0x68, 0x58, - 0xe8, 0x93, 0x30, 0x19, 0xf2, 0xa0, 0x84, 0x4b, 0x0d, 0x27, 0x64, 0x6e, 0x2e, 0x62, 0x15, 0xb2, - 0xb8, 0x4e, 0xd5, 0x04, 0x0c, 0x77, 0x60, 0xa3, 0x15, 0xd9, 0x2a, 0x8b, 0xa0, 0xc8, 0x17, 0xe6, - 0xe5, 0xb8, 0x45, 0x91, 0xf1, 0xf7, 0x54, 0x72, 0xf8, 0xd9, 0xc0, 0x6b, 0x35, 0xd1, 0x67, 0x00, - 0xe8, 0xf2, 0x11, 0x72, 0x88, 0xe1, 0xec, 0xc3, 0x93, 0x9e, 0x2a, 0xf5, 0x54, 0xab, 0x5c, 0xe6, - 0x9b, 0x5a, 0x52, 0x44, 0xb0, 0x46, 0xd0, 0xfe, 0xe1, 0x01, 0x78, 0xb4, 0xcb, 0x19, 0x89, 0x16, - 0x4c, 0x3d, 0xec, 0x93, 0xc9, 0xc7, 0xf5, 0x6c, 0x6a, 0x65, 0xe3, 0xb5, 0x9d, 0x58, 0x8a, 0xb9, - 0xf7, 0xbc, 0x14, 0x7f, 0xc0, 0xd2, 0xc4, 0x1e, 0xdc, 0x56, 0xf3, 0x13, 0x87, 0x3c, 0xfb, 0x8f, - 0x50, 0x0e, 0xb2, 0x99, 0x22, 0x4c, 0x78, 0xb6, 0xef, 0xee, 0xf4, 0x2d, 0x5d, 0x38, 0x59, 0x29, - 0xf1, 0x6f, 0x5b, 0x70, 0xbe, 0x6b, 0x70, 0x8e, 0x6f, 0x40, 0x86, 0xc1, 0xfe, 0x82, 0x05, 0x8f, - 0xa5, 0xd6, 0x30, 0xcc, 0x8c, 0xae, 0x42, 0xb1, 0x46, 0x0b, 0x35, 0xff, 0xca, 0xd8, 0xf1, 0x5c, - 0x02, 0x70, 0x8c, 0x73, 0xc8, 0xc0, 0x23, 0xbf, 0x6a, 0x41, 0xc7, 0xa6, 0x3f, 0x81, 0xdb, 0xa7, - 0x6c, 0xde, 0x3e, 0x1f, 0xee, 0x67, 0x34, 0x33, 0x2e, 0x9e, 0x3f, 0x9e, 0x80, 0x33, 0x19, 0xfe, - 0x45, 0xbb, 0x30, 0xb5, 0x55, 0x23, 0xa6, 0xe7, 0x6a, 0xb7, 0xf8, 0x2f, 0x5d, 0xdd, 0x5c, 0x59, - 0x4e, 0xd2, 0xa9, 0x0e, 0x14, 0xdc, 0xd9, 0x04, 0xfa, 0x82, 0x05, 0xa7, 0x9c, 0x7b, 0xe1, 0x32, - 0xe5, 0x22, 0xdc, 0xda, 0x62, 0xc3, 0xaf, 0xed, 0xd0, 0x23, 0x5a, 0x6e, 0x84, 0xe7, 0x53, 0x25, - 0x3b, 0x77, 0xab, 0x1d, 0xf8, 0x46, 0xf3, 0x2c, 0x49, 0x6b, 0x1a, 0x16, 0x4e, 0x6d, 0x0b, 0x61, - 0x11, 0xb9, 0x9f, 0xbe, 0x51, 0xba, 0xf8, 0x56, 0xa7, 0x39, 0x82, 0xf1, 0x6b, 0x51, 0x42, 0xb0, - 0xa2, 0x83, 0x3e, 0x07, 0xc5, 0x2d, 0xe9, 0x9d, 0x99, 0x72, 0xed, 0xc6, 0x03, 0xd9, 0xdd, 0x67, - 0x95, 0xab, 0x67, 0x15, 0x12, 0x8e, 0x89, 0xa2, 0xd7, 0x20, 0xef, 0x6d, 0x86, 0xdd, 0xf2, 0x9c, - 0x26, 0xec, 0xf0, 0x78, 0x04, 0x83, 0xb5, 0x95, 0x2a, 0xa6, 0x15, 0xd1, 0x75, 0xc8, 0x07, 0x1b, - 0x75, 0x21, 0x96, 0x4c, 0xdd, 0xa4, 0x78, 0xb1, 0x94, 0xd1, 0x2b, 0x46, 0x09, 0x2f, 0x96, 0x30, - 0x25, 0x81, 0x2a, 0x30, 0xc8, 0x9c, 0x72, 0xc4, 0x25, 0x97, 0xca, 0xce, 0x77, 0x71, 0x6e, 0xe3, - 0x61, 0x0e, 0x18, 0x02, 0xe6, 0x84, 0xd0, 0x3a, 0x0c, 0xd5, 0x58, 0x4e, 0x4c, 0x11, 0xf1, 0xed, - 0x63, 0xa9, 0x02, 0xc8, 0x2e, 0xc9, 0x42, 0x85, 0x3c, 0x8e, 0x61, 0x60, 0x41, 0x8b, 0x51, 0x25, - 0xad, 0xed, 0xcd, 0x50, 0xe4, 0x70, 0x4e, 0xa7, 0xda, 0x25, 0x07, 0xae, 0xa0, 0xca, 0x30, 0xb0, - 0xa0, 0x85, 0x5e, 0x86, 0xdc, 0x66, 0x4d, 0x38, 0xdc, 0xa4, 0x4a, 0x22, 0xcd, 0x20, 0x14, 0x8b, - 0x43, 0x0f, 0xf6, 0xe7, 0x72, 0x2b, 0x4b, 0x38, 0xb7, 0x59, 0x43, 0x6b, 0x30, 0xbc, 0xc9, 0xdd, - 0xd6, 0x85, 0xb0, 0xf1, 0xf1, 0x74, 0x8f, 0xfa, 0x0e, 0xcf, 0x76, 0xee, 0x6b, 0x22, 0x00, 0x58, - 0x12, 0x61, 0x81, 0xf0, 0x95, 0xfb, 0xbd, 0x08, 0x8e, 0x36, 0x7f, 0xb8, 0x90, 0x09, 0x9c, 0xe9, - 0x88, 0x9d, 0xf8, 0xb1, 0x46, 0x91, 0xae, 0x6a, 0x47, 0x26, 0xd2, 0x17, 0x61, 0x62, 0x52, 0x57, - 0xb5, 0xca, 0xb6, 0xdf, 0x6d, 0x55, 0x2b, 0x24, 0x1c, 0x13, 0x45, 0x3b, 0x30, 0xb6, 0x1b, 0xb6, - 0xb6, 0x89, 0xdc, 0xd2, 0x2c, 0x6a, 0x4c, 0xc6, 0xbd, 0x7c, 0x47, 0x20, 0xba, 0x41, 0xd4, 0x76, - 0x1a, 0x1d, 0xa7, 0x10, 0xd3, 0xe9, 0xdf, 0xd1, 0x89, 0x61, 0x93, 0x36, 0x1d, 0xfe, 0x77, 0xda, - 0xfe, 0xc6, 0x5e, 0x44, 0x44, 0x4c, 0xb3, 0xd4, 0xe1, 0x7f, 0x83, 0xa3, 0x74, 0x0e, 0xbf, 0x00, - 0x60, 0x49, 0x04, 0xdd, 0x11, 0xc3, 0xc3, 0x4e, 0xcf, 0xc9, 0xec, 0xc0, 0xa3, 0x0b, 0x12, 0x29, - 0x63, 0x50, 0xd8, 0x69, 0x19, 0x93, 0x62, 0xa7, 0x64, 0x6b, 0xdb, 0x8f, 0x7c, 0x2f, 0x71, 0x42, - 0x4f, 0x65, 0x9f, 0x92, 0x95, 0x14, 0xfc, 0xce, 0x53, 0x32, 0x0d, 0x0b, 0xa7, 0xb6, 0x85, 0xea, - 0x30, 0xde, 0xf2, 0x83, 0xe8, 0x9e, 0x1f, 0xc8, 0xf5, 0x85, 0xba, 0x08, 0x4b, 0x0c, 0x4c, 0xd1, - 0x22, 0x0b, 0x17, 0x68, 0x42, 0x70, 0x82, 0x26, 0xfa, 0x14, 0x0c, 0x87, 0x35, 0xa7, 0x41, 0xca, - 0xb7, 0x66, 0xa6, 0xb3, 0xaf, 0x9f, 0x2a, 0x47, 0xc9, 0x58, 0x5d, 0x3c, 0x6a, 0x3e, 0x47, 0xc1, - 0x92, 0x1c, 0x5a, 0x81, 0x41, 0x96, 0xe8, 0x8c, 0x05, 0xe0, 0xcb, 0x88, 0x9f, 0xda, 0x61, 0x15, - 0xcd, 0xcf, 0x26, 0x56, 0x8c, 0x79, 0x75, 0xba, 0x07, 0xc4, 0x9b, 0xc1, 0x0f, 0x67, 0x4e, 0x67, - 0xef, 0x01, 0xf1, 0xd4, 0xb8, 0x55, 0xed, 0xb6, 0x07, 0x14, 0x12, 0x8e, 0x89, 0xd2, 0x93, 0x99, - 0x9e, 0xa6, 0x67, 0xba, 0x98, 0xf3, 0x64, 0x9e, 0xa5, 0xec, 0x64, 0xa6, 0x27, 0x29, 0x25, 0x61, - 0xff, 0xde, 0x70, 0x27, 0xcf, 0xc2, 0x5e, 0x99, 0xdf, 0x65, 0x75, 0x28, 0x20, 0x3f, 0xde, 0xaf, - 0xd0, 0xeb, 0x08, 0x59, 0xf0, 0x2f, 0x58, 0x70, 0xa6, 0x95, 0xfa, 0x21, 0x82, 0x01, 0xe8, 0x4f, - 0x76, 0xc6, 0x3f, 0x5d, 0x05, 0x6b, 0x4c, 0x87, 0xe3, 0x8c, 0x96, 0x92, 0xcf, 0x9c, 0xfc, 0x7b, - 0x7e, 0xe6, 0xac, 0x42, 0x81, 0x31, 0x99, 0x3d, 0x72, 0x44, 0x27, 0x5f, 0x7b, 0x8c, 0x95, 0x58, - 0x12, 0x15, 0xb1, 0x22, 0x81, 0x7e, 0xd0, 0x82, 0xf3, 0xc9, 0xae, 0x63, 0xc2, 0xc0, 0x22, 0xc2, - 0x23, 0x7f, 0xe0, 0xae, 0x88, 0xef, 0xef, 0xe0, 0xff, 0x0d, 0xe4, 0x83, 0x5e, 0x08, 0xb8, 0x7b, - 0x63, 0xa8, 0x94, 0xf2, 0xc2, 0x1e, 0x32, 0xb5, 0x0a, 0x7d, 0xbc, 0xb2, 0x9f, 0x87, 0xd1, 0xa6, - 0xdf, 0xf6, 0x22, 0x61, 0xfd, 0x23, 0x2c, 0x11, 0x98, 0x06, 0x7e, 0x55, 0x2b, 0xc7, 0x06, 0x56, - 0xe2, 0x6d, 0x5e, 0x78, 0xe8, 0xb7, 0xf9, 0x5b, 0x30, 0xea, 0x69, 0xe6, 0xaa, 0x82, 0x1f, 0xb8, - 0x9c, 0x1d, 0x9d, 0x55, 0x37, 0x6e, 0xe5, 0xbd, 0xd4, 0x4b, 0xb0, 0x41, 0xed, 0x64, 0x1f, 0x7c, - 0x5f, 0xb6, 0x52, 0x98, 0x7a, 0x2e, 0x02, 0x78, 0xd5, 0x14, 0x01, 0x5c, 0x4e, 0x8a, 0x00, 0x3a, - 0x24, 0xca, 0xc6, 0xeb, 0xbf, 0xff, 0xe4, 0x33, 0xfd, 0x86, 0x30, 0xb4, 0x1b, 0x70, 0xb1, 0xd7, - 0xb5, 0xc4, 0xcc, 0xc0, 0xea, 0x4a, 0x7f, 0x18, 0x9b, 0x81, 0xd5, 0xcb, 0x25, 0xcc, 0x20, 0xfd, - 0x06, 0xc7, 0xb1, 0xff, 0x9b, 0x05, 0xf9, 0x8a, 0x5f, 0x3f, 0x81, 0x07, 0xef, 0x27, 0x8c, 0x07, - 0xef, 0xa3, 0xe9, 0x17, 0x62, 0x3d, 0x53, 0x1e, 0xbe, 0x9c, 0x90, 0x87, 0x9f, 0xcf, 0x22, 0xd0, - 0x5d, 0xfa, 0xfd, 0x93, 0x79, 0x18, 0xa9, 0xf8, 0x75, 0x65, 0x83, 0xfd, 0xeb, 0x0f, 0x63, 0x83, - 0x9d, 0x99, 0x42, 0x41, 0xa3, 0xcc, 0xac, 0xc7, 0xa4, 0xe3, 0xe8, 0x37, 0x98, 0x29, 0xf6, 0x5d, - 0xe2, 0x6e, 0x6d, 0x47, 0xa4, 0x9e, 0xfc, 0x9c, 0x93, 0x33, 0xc5, 0xfe, 0xaf, 0x16, 0x4c, 0x24, - 0x5a, 0x47, 0x0d, 0x18, 0x6b, 0xe8, 0xd2, 0x56, 0xb1, 0x4e, 0x1f, 0x4a, 0x50, 0x2b, 0x4c, 0x59, - 0xb5, 0x22, 0x6c, 0x12, 0x47, 0xf3, 0x00, 0x4a, 0xfd, 0x28, 0xc5, 0x7a, 0x8c, 0xeb, 0x57, 0xfa, - 0xc9, 0x10, 0x6b, 0x18, 0xe8, 0x05, 0x18, 0x89, 0xfc, 0x96, 0xdf, 0xf0, 0xb7, 0xf6, 0x6e, 0x10, - 0x19, 0x8e, 0x49, 0x19, 0xa8, 0xad, 0xc7, 0x20, 0xac, 0xe3, 0xd9, 0x3f, 0x9d, 0xe7, 0x1f, 0xea, - 0x45, 0xee, 0x07, 0x6b, 0xf2, 0xfd, 0xbd, 0x26, 0xbf, 0x6a, 0xc1, 0x24, 0x6d, 0x9d, 0xd9, 0xc0, - 0xc8, 0xcb, 0x56, 0x45, 0x65, 0xb6, 0xba, 0x44, 0x65, 0xbe, 0x4c, 0xcf, 0xae, 0xba, 0xdf, 0x8e, + 0x8a, 0xd3, 0x17, 0x40, 0x9b, 0x7d, 0x5d, 0xc8, 0x54, 0x0a, 0xe2, 0x05, 0xc0, 0x3f, 0x38, 0xc4, + 0x12, 0x46, 0x5f, 0x8e, 0xf5, 0x1d, 0xa7, 0xbd, 0xd8, 0x89, 0x76, 0xca, 0x6e, 0x58, 0xf7, 0xf7, + 0x48, 0xb0, 0xcf, 0x1e, 0xfd, 0x23, 0xf1, 0xcb, 0x51, 0x01, 0x96, 0xaf, 0x2f, 0x56, 0x29, 0x26, + 0x4e, 0xd7, 0x41, 0x8b, 0x30, 0x29, 0x0b, 0x6b, 0x24, 0x64, 0x57, 0xd8, 0x28, 0x23, 0xa3, 0x9c, + 0xa5, 0x44, 0xb1, 0x22, 0x92, 0xc4, 0x37, 0x39, 0x69, 0x38, 0x0e, 0x4e, 0xfa, 0x23, 0x30, 0xee, + 0x7a, 0x6e, 0xe4, 0x3a, 0x91, 0xcf, 0xf5, 0x61, 0xfc, 0x7d, 0xcf, 0x04, 0xfd, 0x15, 0x1d, 0x80, + 0x4d, 0x3c, 0xfb, 0x3f, 0x0f, 0xc0, 0x34, 0x9b, 0xb6, 0xf7, 0x56, 0xd8, 0x37, 0xd3, 0x0a, 0xbb, + 0x9d, 0x5e, 0x61, 0xc7, 0xf1, 0x44, 0x78, 0xe8, 0x65, 0xf6, 0x39, 0x28, 0x29, 0xff, 0x30, 0xe9, + 0x20, 0x6a, 0xe5, 0x38, 0x88, 0xf6, 0xe6, 0x3e, 0xa4, 0x89, 0x5d, 0x31, 0xd3, 0xc4, 0xee, 0xaf, + 0x5b, 0x10, 0x2b, 0x78, 0xd0, 0x75, 0x28, 0xb5, 0x7d, 0x66, 0x39, 0x1a, 0x48, 0x73, 0xec, 0xc7, + 0x33, 0x2f, 0x2a, 0x7e, 0x29, 0xf2, 0x8f, 0xaf, 0xca, 0x1a, 0x38, 0xae, 0x8c, 0x96, 0x60, 0xb8, + 0x1d, 0x90, 0x5a, 0xc4, 0x62, 0xa8, 0xf4, 0xa4, 0xc3, 0xd7, 0x08, 0xc7, 0xc7, 0xb2, 0xa2, 0xfd, + 0xf3, 0x16, 0x00, 0xb7, 0x62, 0x73, 0xbc, 0x6d, 0x72, 0x02, 0x52, 0xeb, 0x32, 0x0c, 0x84, 0x6d, + 0x52, 0xef, 0x66, 0xd3, 0x1b, 0xf7, 0xa7, 0xd6, 0x26, 0xf5, 0x78, 0xc0, 0xe9, 0x3f, 0xcc, 0x6a, + 0xdb, 0xdf, 0x0d, 0x30, 0x11, 0xa3, 0x55, 0x22, 0xd2, 0x42, 0xcf, 0x1a, 0x31, 0x15, 0xce, 0x26, + 0x62, 0x2a, 0x94, 0x18, 0xb6, 0x26, 0x20, 0xfd, 0x1c, 0x14, 0x5b, 0xce, 0x7d, 0x21, 0x01, 0x7b, + 0xba, 0x7b, 0x37, 0x28, 0xfd, 0x85, 0x35, 0xe7, 0x3e, 0x7f, 0x24, 0x3e, 0x2d, 0x17, 0xc8, 0x9a, + 0x73, 0xff, 0x90, 0x5b, 0xee, 0xb2, 0x43, 0xea, 0xa6, 0x1b, 0x46, 0x5f, 0xf8, 0x4f, 0xf1, 0x7f, + 0xb6, 0xec, 0x68, 0x23, 0xac, 0x2d, 0xd7, 0x13, 0x06, 0x5a, 0x7d, 0xb5, 0xe5, 0x7a, 0xc9, 0xb6, + 0x5c, 0xaf, 0x8f, 0xb6, 0x5c, 0x0f, 0xbd, 0x0d, 0xc3, 0xc2, 0x7e, 0x52, 0xc4, 0x30, 0xba, 0xda, + 0x47, 0x7b, 0xc2, 0xfc, 0x92, 0xb7, 0x79, 0x55, 0x3e, 0x82, 0x45, 0x69, 0xcf, 0x76, 0x65, 0x83, + 0xe8, 0xaf, 0x58, 0x30, 0x21, 0x7e, 0x63, 0xf2, 0x56, 0x87, 0x84, 0x91, 0xe0, 0x3d, 0x3f, 0xdc, + 0x7f, 0x1f, 0x44, 0x45, 0xde, 0x95, 0x0f, 0xcb, 0x63, 0xd6, 0x04, 0xf6, 0xec, 0x51, 0xa2, 0x17, + 0xe8, 0xef, 0x59, 0x70, 0xaa, 0xe5, 0xdc, 0xe7, 0x2d, 0xf2, 0x32, 0xec, 0x44, 0xae, 0x2f, 0xec, + 0x10, 0x3e, 0xda, 0xdf, 0xf4, 0xa7, 0xaa, 0xf3, 0x4e, 0x4a, 0x65, 0xe9, 0xa9, 0x2c, 0x94, 0x9e, + 0x5d, 0xcd, 0xec, 0xd7, 0xdc, 0x16, 0x8c, 0xc8, 0xf5, 0x96, 0x21, 0x6a, 0x28, 0xeb, 0x8c, 0xf5, + 0x91, 0xcd, 0x57, 0xf5, 0x58, 0x05, 0xb4, 0x1d, 0xb1, 0xd6, 0x1e, 0x69, 0x3b, 0x9f, 0x83, 0x31, + 0x7d, 0x8d, 0x3d, 0xd2, 0xb6, 0xde, 0x82, 0x99, 0x8c, 0xb5, 0xf4, 0x48, 0x9b, 0xbc, 0x07, 0x67, + 0x73, 0xd7, 0xc7, 0xa3, 0x6c, 0xd8, 0xfe, 0x39, 0x4b, 0x3f, 0x07, 0x4f, 0x40, 0x75, 0xb0, 0x6c, + 0xaa, 0x0e, 0x2e, 0x74, 0xdf, 0x39, 0x39, 0xfa, 0x83, 0x37, 0xf5, 0x4e, 0xd3, 0x53, 0x1d, 0xbd, + 0x0e, 0x43, 0x4d, 0x5a, 0x22, 0xad, 0x70, 0xed, 0xde, 0x3b, 0x32, 0xe6, 0xa5, 0x58, 0x79, 0x88, + 0x05, 0x05, 0xfb, 0x97, 0x2c, 0x18, 0x38, 0x81, 0x91, 0xc0, 0xe6, 0x48, 0x3c, 0x9b, 0x4b, 0x5a, + 0x84, 0x57, 0x5e, 0xc0, 0xce, 0xbd, 0x95, 0xfb, 0x11, 0xf1, 0x42, 0xf6, 0x54, 0xcc, 0x1c, 0x98, + 0xff, 0x0b, 0x66, 0x6e, 0xfa, 0x4e, 0x63, 0xc9, 0x69, 0x3a, 0x5e, 0x9d, 0x04, 0x15, 0x6f, 0xfb, + 0x48, 0x16, 0xe4, 0x85, 0x5e, 0x16, 0xe4, 0xf6, 0x0e, 0x20, 0xbd, 0x01, 0xe1, 0x8a, 0x83, 0x61, + 0xd8, 0xe5, 0x4d, 0x89, 0xe1, 0x7f, 0x32, 0x9b, 0x35, 0x4b, 0xf5, 0x4c, 0x73, 0x32, 0xe1, 0x05, + 0x58, 0x12, 0xb2, 0x5f, 0x86, 0x4c, 0x7f, 0xfe, 0xde, 0x62, 0x03, 0xfb, 0x93, 0x30, 0xcd, 0x6a, + 0x1e, 0xf1, 0x49, 0x6b, 0x27, 0xa4, 0x92, 0x19, 0x91, 0xfe, 0xec, 0x2f, 0x5a, 0x30, 0xb9, 0x9e, + 0x08, 0x80, 0x76, 0x99, 0xe9, 0x31, 0x33, 0x84, 0xe1, 0x35, 0x56, 0x8a, 0x05, 0xf4, 0xd8, 0x65, + 0x50, 0x7f, 0x69, 0x41, 0x1c, 0x62, 0xe3, 0x04, 0x18, 0xaf, 0x65, 0x83, 0xf1, 0xca, 0x94, 0x8d, + 0xa8, 0xee, 0xe4, 0xf1, 0x5d, 0xe8, 0x86, 0x0a, 0x3e, 0xd5, 0x45, 0x2c, 0x12, 0x93, 0xe1, 0xa1, + 0x8a, 0x26, 0xcc, 0x08, 0x55, 0x32, 0x1c, 0x95, 0xfd, 0x1f, 0x0a, 0x80, 0x14, 0x6e, 0xdf, 0xc1, + 0xb1, 0xd2, 0x35, 0x8e, 0x27, 0x38, 0xd6, 0x1e, 0x20, 0xa6, 0x89, 0x0f, 0x1c, 0x2f, 0xe4, 0x64, + 0x5d, 0x21, 0x75, 0x3b, 0x9a, 0x9a, 0x7f, 0x4e, 0x34, 0x89, 0x6e, 0xa6, 0xa8, 0xe1, 0x8c, 0x16, + 0x34, 0x0b, 0x8b, 0xc1, 0x7e, 0x2d, 0x2c, 0x86, 0x7a, 0xb8, 0xdb, 0xfd, 0xac, 0x05, 0xe3, 0x6a, + 0x98, 0xde, 0x25, 0xc6, 0xf0, 0xaa, 0x3f, 0x39, 0x47, 0x5f, 0x55, 0xeb, 0x32, 0xbb, 0x12, 0xbe, + 0x95, 0xb9, 0x4d, 0x3a, 0x4d, 0xf7, 0x6d, 0xa2, 0x42, 0x13, 0xce, 0x0b, 0x37, 0x48, 0x51, 0x7a, + 0x78, 0x30, 0x3f, 0xae, 0xfe, 0xf1, 0x50, 0xc8, 0x71, 0x15, 0xfb, 0x27, 0xe9, 0x66, 0x37, 0x97, + 0x22, 0x7a, 0x09, 0x06, 0xdb, 0x3b, 0x4e, 0x48, 0x12, 0x4e, 0x43, 0x83, 0x55, 0x5a, 0x78, 0x78, + 0x30, 0x3f, 0xa1, 0x2a, 0xb0, 0x12, 0xcc, 0xb1, 0xfb, 0x0f, 0x39, 0x96, 0x5e, 0x9c, 0x3d, 0x43, + 0x8e, 0xfd, 0xa9, 0x05, 0x03, 0xeb, 0x7e, 0xe3, 0x24, 0x8e, 0x80, 0xd7, 0x8c, 0x23, 0xe0, 0x5c, + 0x5e, 0x94, 0xfa, 0xdc, 0xdd, 0xbf, 0x9a, 0xd8, 0xfd, 0x17, 0x72, 0x29, 0x74, 0xdf, 0xf8, 0x2d, + 0x18, 0x65, 0xb1, 0xef, 0x85, 0x83, 0xd4, 0x0b, 0xc6, 0x86, 0x9f, 0x4f, 0x6c, 0xf8, 0x49, 0x0d, + 0x55, 0xdb, 0xe9, 0x4f, 0xc1, 0xb0, 0xf0, 0xb8, 0x49, 0x7a, 0x9f, 0x0a, 0x5c, 0x2c, 0xe1, 0xf6, + 0x8f, 0x17, 0xc1, 0x88, 0xb5, 0x8f, 0x7e, 0xc5, 0x82, 0x85, 0x80, 0x5b, 0xe2, 0x36, 0xca, 0x9d, + 0xc0, 0xf5, 0xb6, 0x6b, 0xf5, 0x1d, 0xd2, 0xe8, 0x34, 0x5d, 0x6f, 0xbb, 0xb2, 0xed, 0xf9, 0xaa, + 0x78, 0xe5, 0x3e, 0xa9, 0x77, 0x98, 0xfa, 0xaa, 0x47, 0x60, 0x7f, 0x65, 0xd1, 0xfe, 0xfc, 0x83, + 0x83, 0xf9, 0x05, 0x7c, 0x24, 0xda, 0xf8, 0x88, 0x7d, 0x41, 0xbf, 0x65, 0xc1, 0x55, 0x1e, 0x82, + 0xbe, 0xff, 0xfe, 0x77, 0x79, 0xe7, 0x56, 0x25, 0xa9, 0x98, 0xc8, 0x06, 0x09, 0x5a, 0x4b, 0x1f, + 0x11, 0x03, 0x7a, 0xb5, 0x7a, 0xb4, 0xb6, 0xf0, 0x51, 0x3b, 0x67, 0xff, 0xd3, 0x22, 0x8c, 0x8b, + 0xd0, 0x54, 0xe2, 0x0e, 0x78, 0xc9, 0x58, 0x12, 0x4f, 0x24, 0x96, 0xc4, 0xb4, 0x81, 0x7c, 0x3c, + 0xc7, 0x7f, 0x08, 0xd3, 0xf4, 0x70, 0xbe, 0x4e, 0x9c, 0x20, 0xda, 0x24, 0x0e, 0x37, 0x9c, 0x2a, + 0x1e, 0xf9, 0xf4, 0x57, 0x82, 0xb5, 0x9b, 0x49, 0x62, 0x38, 0x4d, 0xff, 0x9b, 0xe9, 0xce, 0xf1, + 0x60, 0x2a, 0x15, 0x5d, 0xec, 0x53, 0x50, 0x52, 0xee, 0x22, 0xe2, 0xd0, 0xe9, 0x1e, 0xa4, 0x2f, + 0x49, 0x81, 0x0b, 0xbf, 0x62, 0x57, 0xa5, 0x98, 0x9c, 0xfd, 0xf7, 0x0b, 0x46, 0x83, 0x7c, 0x12, + 0xd7, 0x61, 0xc4, 0x09, 0x43, 0x77, 0xdb, 0x23, 0x0d, 0xb1, 0x63, 0xdf, 0x9f, 0xb7, 0x63, 0x8d, + 0x66, 0x98, 0xcb, 0xce, 0xa2, 0xa8, 0x89, 0x15, 0x0d, 0x74, 0x9d, 0x9b, 0xa7, 0xed, 0xc9, 0x97, + 0x5a, 0x7f, 0xd4, 0x40, 0x1a, 0xb0, 0xed, 0x11, 0x2c, 0xea, 0xa3, 0x4f, 0x73, 0xfb, 0xc1, 0x1b, + 0x9e, 0x7f, 0xcf, 0xbb, 0xe6, 0xfb, 0x32, 0xfc, 0x43, 0x7f, 0x04, 0xa7, 0xa5, 0xd5, 0xa0, 0xaa, + 0x8e, 0x4d, 0x6a, 0xfd, 0x85, 0xeb, 0xfc, 0x3c, 0xcc, 0x50, 0xd2, 0xa6, 0x77, 0x76, 0x88, 0x08, + 0x4c, 0x8a, 0xb8, 0x67, 0xb2, 0x4c, 0x8c, 0x5d, 0xe6, 0x23, 0xcc, 0xac, 0x1d, 0x4b, 0x80, 0x6f, + 0x98, 0x24, 0x70, 0x92, 0xa6, 0xfd, 0x53, 0x16, 0x30, 0x4f, 0xd5, 0x13, 0xe0, 0x47, 0x3e, 0x66, + 0xf2, 0x23, 0xb3, 0x79, 0x83, 0x9c, 0xc3, 0x8a, 0xbc, 0xc8, 0x57, 0x56, 0x35, 0xf0, 0xef, 0xef, + 0x0b, 0xa3, 0x8f, 0xde, 0xef, 0x0f, 0xfb, 0x7f, 0x59, 0xfc, 0x10, 0x53, 0xce, 0x1c, 0xe8, 0xdb, + 0x61, 0xa4, 0xee, 0xb4, 0x9d, 0x3a, 0x4f, 0x0c, 0x93, 0x2b, 0x8b, 0x33, 0x2a, 0x2d, 0x2c, 0x8b, + 0x1a, 0x5c, 0xb6, 0x24, 0xe3, 0xe7, 0x8d, 0xc8, 0xe2, 0x9e, 0xf2, 0x24, 0xd5, 0xe4, 0xdc, 0x2e, + 0x8c, 0x1b, 0xc4, 0x1e, 0xa9, 0x20, 0xe2, 0xdb, 0xf9, 0x15, 0xab, 0xe2, 0x3d, 0xb6, 0x60, 0xda, + 0xd3, 0xfe, 0xd3, 0x0b, 0x45, 0x3e, 0x2e, 0xdf, 0xdf, 0xeb, 0x12, 0x65, 0xb7, 0x8f, 0xe6, 0x04, + 0x9b, 0x20, 0x83, 0xd3, 0x94, 0xed, 0x9f, 0xb0, 0xe0, 0x31, 0x1d, 0x51, 0xf3, 0xb3, 0xe9, 0x25, + 0xdd, 0x2f, 0xc3, 0x88, 0xdf, 0x26, 0x81, 0x13, 0xf9, 0x81, 0xb8, 0x35, 0xae, 0xc8, 0x41, 0xbf, + 0x25, 0xca, 0x0f, 0x45, 0x58, 0x75, 0x49, 0x5d, 0x96, 0x63, 0x55, 0x93, 0xbe, 0x3e, 0xd9, 0x60, + 0x84, 0xc2, 0xa3, 0x8a, 0x9d, 0x01, 0x4c, 0xd1, 0x1d, 0x62, 0x01, 0xb1, 0xbf, 0x66, 0xf1, 0x85, + 0xa5, 0x77, 0x1d, 0xbd, 0x05, 0x53, 0x2d, 0x27, 0xaa, 0xef, 0xac, 0xdc, 0x6f, 0x07, 0x5c, 0x57, + 0x22, 0xc7, 0xe9, 0xe9, 0x5e, 0xe3, 0xa4, 0x7d, 0x64, 0x6c, 0x12, 0xb9, 0x96, 0x20, 0x86, 0x53, + 0xe4, 0xd1, 0x26, 0x8c, 0xb2, 0x32, 0xe6, 0x2c, 0x18, 0x76, 0x63, 0x0d, 0xf2, 0x5a, 0x53, 0xb6, + 0x02, 0x6b, 0x31, 0x1d, 0xac, 0x13, 0xb5, 0x7f, 0xa6, 0xc8, 0x77, 0x3b, 0x63, 0xe5, 0x9f, 0x82, + 0xe1, 0xb6, 0xdf, 0x58, 0xae, 0x94, 0xb1, 0x98, 0x05, 0x75, 0x8d, 0x54, 0x79, 0x31, 0x96, 0x70, + 0x74, 0x05, 0x46, 0xc4, 0x4f, 0xa9, 0xdb, 0x62, 0x67, 0xb3, 0xc0, 0x0b, 0xb1, 0x82, 0xa2, 0xe7, + 0x01, 0xda, 0x81, 0xbf, 0xe7, 0x36, 0x58, 0x10, 0x8b, 0xa2, 0x69, 0xe6, 0x53, 0x55, 0x10, 0xac, + 0x61, 0xa1, 0x57, 0x61, 0xbc, 0xe3, 0x85, 0x9c, 0x1d, 0xd1, 0x42, 0xd6, 0x2a, 0x03, 0x94, 0xdb, + 0x3a, 0x10, 0x9b, 0xb8, 0x68, 0x11, 0x86, 0x22, 0x87, 0x99, 0xad, 0x0c, 0xe6, 0x9b, 0xcd, 0x6e, + 0x50, 0x0c, 0x3d, 0x07, 0x09, 0xad, 0x80, 0x45, 0x45, 0xf4, 0x29, 0xe9, 0xb7, 0xcb, 0x0f, 0x76, + 0x61, 0xaf, 0xde, 0xdf, 0x25, 0xa0, 0x79, 0xed, 0x0a, 0x3b, 0x78, 0x83, 0x16, 0x7a, 0x05, 0x80, + 0xdc, 0x8f, 0x48, 0xe0, 0x39, 0x4d, 0x65, 0x15, 0xa6, 0xf8, 0x82, 0xb2, 0xbf, 0xee, 0x47, 0xb7, + 0x43, 0xb2, 0xa2, 0x30, 0xb0, 0x86, 0x6d, 0xff, 0x56, 0x09, 0x20, 0xe6, 0xdb, 0xd1, 0xdb, 0xa9, + 0x83, 0xeb, 0x99, 0xee, 0x9c, 0xfe, 0xf1, 0x9d, 0x5a, 0xe8, 0x7b, 0x2c, 0x18, 0x75, 0x9a, 0x4d, + 0xbf, 0xee, 0xf0, 0xa0, 0xc2, 0x85, 0xee, 0x07, 0xa7, 0x68, 0x7f, 0x31, 0xae, 0xc1, 0xbb, 0xf0, + 0x82, 0x5c, 0xa1, 0x1a, 0xa4, 0x67, 0x2f, 0xf4, 0x86, 0xd1, 0x87, 0xe4, 0x53, 0xb1, 0x68, 0x0c, + 0xa5, 0x7a, 0x2a, 0x96, 0xd8, 0x1d, 0xa1, 0xbf, 0x12, 0x6f, 0x1b, 0xaf, 0xc4, 0x81, 0x7c, 0xc7, + 0x44, 0x83, 0x7d, 0xed, 0xf5, 0x40, 0x44, 0x55, 0x3d, 0x48, 0xc1, 0x60, 0xbe, 0x17, 0xa0, 0xf6, + 0x4e, 0xea, 0x11, 0xa0, 0xe0, 0x73, 0x30, 0xd9, 0x30, 0x99, 0x00, 0xb1, 0x12, 0x9f, 0xcc, 0xa3, + 0x9b, 0xe0, 0x19, 0xe2, 0x6b, 0x3f, 0x01, 0xc0, 0x49, 0xc2, 0xa8, 0xca, 0x63, 0x56, 0x54, 0xbc, + 0x2d, 0x5f, 0xf8, 0x4c, 0xd8, 0xb9, 0x73, 0xb9, 0x1f, 0x46, 0xa4, 0x45, 0x31, 0xe3, 0xdb, 0x7d, + 0x5d, 0xd4, 0xc5, 0x8a, 0x0a, 0x7a, 0x1d, 0x86, 0x98, 0x1b, 0x58, 0x38, 0x3b, 0x92, 0x2f, 0x2b, + 0x36, 0x83, 0xb0, 0xc5, 0x1b, 0x92, 0xfd, 0x0d, 0xb1, 0xa0, 0x80, 0xae, 0x4b, 0x27, 0xcb, 0xb0, + 0xe2, 0xdd, 0x0e, 0x09, 0x73, 0xb2, 0x2c, 0x2d, 0xbd, 0x3f, 0xf6, 0x9f, 0xe4, 0xe5, 0x99, 0x99, + 0xca, 0x8c, 0x9a, 0x94, 0x8b, 0x12, 0xff, 0x65, 0x02, 0xb4, 0x59, 0xc8, 0xef, 0x9e, 0x99, 0x24, + 0x2d, 0x1e, 0xce, 0x3b, 0x26, 0x09, 0x9c, 0xa4, 0x49, 0x39, 0x52, 0xbe, 0xeb, 0x85, 0xd7, 0x45, + 0xaf, 0xb3, 0x83, 0x3f, 0xc4, 0xd9, 0x6d, 0xc4, 0x4b, 0xb0, 0xa8, 0x7f, 0xa2, 0xec, 0xc1, 0x9c, + 0x07, 0x53, 0xc9, 0x2d, 0xfa, 0x48, 0xd9, 0x91, 0x3f, 0x1c, 0x80, 0x09, 0x73, 0x49, 0xa1, 0xab, + 0x50, 0x12, 0x44, 0x54, 0xd2, 0x02, 0xb5, 0x4b, 0xd6, 0x24, 0x00, 0xc7, 0x38, 0x2c, 0x57, 0x05, + 0xab, 0xae, 0x99, 0xd9, 0xc6, 0xb9, 0x2a, 0x14, 0x04, 0x6b, 0x58, 0xf4, 0x61, 0xb5, 0xe9, 0xfb, + 0x91, 0xba, 0x90, 0xd4, 0xba, 0x5b, 0x62, 0xa5, 0x58, 0x40, 0xe9, 0x45, 0xb4, 0x4b, 0x02, 0x8f, + 0x34, 0xcd, 0xf0, 0xc6, 0xea, 0x22, 0xba, 0xa1, 0x03, 0xb1, 0x89, 0x4b, 0xaf, 0x53, 0x3f, 0x64, + 0x0b, 0x59, 0x3c, 0xdf, 0x62, 0xb3, 0xe5, 0x1a, 0xf7, 0xf3, 0x96, 0x70, 0xf4, 0x49, 0x78, 0x4c, + 0x85, 0x70, 0xc2, 0x5c, 0x0f, 0x21, 0x5b, 0x1c, 0x32, 0xa4, 0x2d, 0x8f, 0x2d, 0x67, 0xa3, 0xe1, + 0xbc, 0xfa, 0xe8, 0x35, 0x98, 0x10, 0x2c, 0xbe, 0xa4, 0x38, 0x6c, 0x9a, 0xc6, 0xdc, 0x30, 0xa0, + 0x38, 0x81, 0x2d, 0x03, 0x34, 0x33, 0x2e, 0x5b, 0x52, 0x18, 0x49, 0x07, 0x68, 0xd6, 0xe1, 0x38, + 0x55, 0x03, 0x2d, 0xc2, 0x24, 0xe7, 0xc1, 0x5c, 0x6f, 0x9b, 0xcf, 0x89, 0x70, 0x8a, 0x52, 0x5b, + 0xea, 0x96, 0x09, 0xc6, 0x49, 0x7c, 0xf4, 0x32, 0x8c, 0x39, 0x41, 0x7d, 0xc7, 0x8d, 0x48, 0x3d, + 0xea, 0x04, 0xdc, 0x5b, 0x4a, 0xb3, 0x2d, 0x5a, 0xd4, 0x60, 0xd8, 0xc0, 0xb4, 0xdf, 0x86, 0x99, + 0x8c, 0x00, 0x10, 0x74, 0xe1, 0x38, 0x6d, 0x57, 0x7e, 0x53, 0xc2, 0x00, 0x79, 0xb1, 0x5a, 0x91, + 0x5f, 0xa3, 0x61, 0xd1, 0xd5, 0xc9, 0x02, 0x45, 0x68, 0xf9, 0x0e, 0xd5, 0xea, 0x5c, 0x95, 0x00, + 0x1c, 0xe3, 0xd8, 0xff, 0xbd, 0x00, 0x93, 0x19, 0xba, 0x15, 0x96, 0x73, 0x2f, 0xf1, 0x48, 0x89, + 0x53, 0xec, 0x99, 0xf1, 0xbe, 0x0b, 0x47, 0x88, 0xf7, 0x5d, 0xec, 0x15, 0xef, 0x7b, 0xe0, 0x9d, + 0xc4, 0xfb, 0x36, 0x47, 0x6c, 0xb0, 0xaf, 0x11, 0xcb, 0x88, 0x11, 0x3e, 0x74, 0xc4, 0x18, 0xe1, + 0xc6, 0xa0, 0x0f, 0xf7, 0x31, 0xe8, 0x3f, 0x5c, 0x80, 0xa9, 0xa4, 0x0d, 0xe4, 0x09, 0xc8, 0x6d, + 0x5f, 0x37, 0xe4, 0xb6, 0x57, 0xfa, 0x71, 0x79, 0xcd, 0x95, 0xe1, 0xe2, 0x84, 0x0c, 0xf7, 0x83, + 0x7d, 0x51, 0xeb, 0x2e, 0xcf, 0xfd, 0x9b, 0x05, 0x38, 0x9d, 0xe9, 0x73, 0x7b, 0x02, 0x63, 0x73, + 0xcb, 0x18, 0x9b, 0x67, 0xfb, 0x76, 0x07, 0xce, 0x1d, 0xa0, 0xbb, 0x89, 0x01, 0xba, 0xda, 0x3f, + 0xc9, 0xee, 0xa3, 0xf4, 0xd5, 0x22, 0x5c, 0xc8, 0xac, 0x17, 0x8b, 0x3d, 0x57, 0x0d, 0xb1, 0xe7, + 0xf3, 0x09, 0xb1, 0xa7, 0xdd, 0xbd, 0xf6, 0xf1, 0xc8, 0x41, 0x85, 0xa3, 0x2b, 0x8b, 0x66, 0xf0, + 0x90, 0x32, 0x50, 0xc3, 0xd1, 0x55, 0x11, 0xc2, 0x26, 0xdd, 0x6f, 0x26, 0xd9, 0xe7, 0xbf, 0xb2, + 0xe0, 0x6c, 0xe6, 0xdc, 0x9c, 0x80, 0xac, 0x6b, 0xdd, 0x94, 0x75, 0x3d, 0xd5, 0xf7, 0x6a, 0xcd, + 0x11, 0x7e, 0xfd, 0xc6, 0x40, 0xce, 0xb7, 0xb0, 0x97, 0xfc, 0x2d, 0x18, 0x75, 0xea, 0x75, 0x12, + 0x86, 0x6b, 0x7e, 0x43, 0x85, 0x34, 0x7e, 0x96, 0xbd, 0xb3, 0xe2, 0xe2, 0xc3, 0x83, 0xf9, 0xb9, + 0x24, 0x89, 0x18, 0x8c, 0x75, 0x0a, 0xe8, 0xd3, 0x30, 0x12, 0x8a, 0x7b, 0x53, 0xcc, 0xfd, 0x0b, + 0x7d, 0x0e, 0x8e, 0xb3, 0x49, 0x9a, 0x66, 0xcc, 0x25, 0x25, 0xa9, 0x50, 0x24, 0xcd, 0xf8, 0x2c, + 0x85, 0x63, 0x8d, 0xcf, 0xf2, 0x3c, 0xc0, 0x9e, 0x7a, 0x0c, 0x24, 0xe5, 0x0f, 0xda, 0x33, 0x41, + 0xc3, 0x42, 0x1f, 0x87, 0xa9, 0x90, 0x07, 0x25, 0x5c, 0x6e, 0x3a, 0x21, 0x73, 0x73, 0x11, 0xab, + 0x90, 0xc5, 0x75, 0xaa, 0x25, 0x60, 0x38, 0x85, 0x8d, 0x56, 0x65, 0xab, 0x2c, 0x82, 0x22, 0x5f, + 0x98, 0x97, 0xe3, 0x16, 0x45, 0xc6, 0xdf, 0x53, 0xc9, 0xe1, 0x67, 0x03, 0xaf, 0xd5, 0x44, 0x9f, + 0x06, 0xa0, 0xcb, 0x47, 0xc8, 0x21, 0x86, 0xf3, 0x0f, 0x4f, 0x7a, 0xaa, 0x34, 0x32, 0xad, 0x72, + 0x99, 0x6f, 0x6a, 0x59, 0x11, 0xc1, 0x1a, 0x41, 0xfb, 0x87, 0x07, 0xe0, 0xf1, 0x2e, 0x67, 0x24, + 0x5a, 0x34, 0xf5, 0xb0, 0x4f, 0x27, 0x1f, 0xd7, 0x73, 0x99, 0x95, 0x8d, 0xd7, 0x76, 0x62, 0x29, + 0x16, 0xde, 0xf1, 0x52, 0xfc, 0x01, 0x4b, 0x13, 0x7b, 0x70, 0x5b, 0xcd, 0x8f, 0x1d, 0xf1, 0xec, + 0x3f, 0x46, 0x39, 0xc8, 0x56, 0x86, 0x30, 0xe1, 0xf9, 0xbe, 0xbb, 0xd3, 0xb7, 0x74, 0xe1, 0x64, + 0xa5, 0xc4, 0xbf, 0x63, 0xc1, 0xf9, 0xae, 0xc1, 0x39, 0xbe, 0x01, 0x19, 0x06, 0xfb, 0x0b, 0x16, + 0x3c, 0x91, 0x59, 0xc3, 0x30, 0x33, 0xba, 0x0a, 0xa5, 0x3a, 0x2d, 0xd4, 0xfc, 0x2b, 0x63, 0xc7, + 0x73, 0x09, 0xc0, 0x31, 0xce, 0x11, 0x03, 0x8f, 0xfc, 0xaa, 0x05, 0xa9, 0x4d, 0x7f, 0x02, 0xb7, + 0x4f, 0xc5, 0xbc, 0x7d, 0xde, 0xdf, 0xcf, 0x68, 0xe6, 0x5c, 0x3c, 0x7f, 0x32, 0x09, 0x67, 0x72, + 0xfc, 0x8b, 0xf6, 0x60, 0x7a, 0xbb, 0x4e, 0x4c, 0xcf, 0xd5, 0x6e, 0xf1, 0x5f, 0xba, 0xba, 0xb9, + 0xb2, 0x9c, 0xa4, 0xd3, 0x29, 0x14, 0x9c, 0x6e, 0x02, 0x7d, 0xc1, 0x82, 0x53, 0xce, 0xbd, 0x70, + 0x85, 0x72, 0x11, 0x6e, 0x7d, 0xa9, 0xe9, 0xd7, 0x77, 0xe9, 0x11, 0x2d, 0x37, 0xc2, 0x8b, 0x99, + 0x92, 0x9d, 0xbb, 0xb5, 0x14, 0xbe, 0xd1, 0x3c, 0x4b, 0xd2, 0x9a, 0x85, 0x85, 0x33, 0xdb, 0x42, + 0x58, 0x44, 0xee, 0xa7, 0x6f, 0x94, 0x2e, 0xbe, 0xd5, 0x59, 0x8e, 0x60, 0xfc, 0x5a, 0x94, 0x10, + 0xac, 0xe8, 0xa0, 0xcf, 0x42, 0x69, 0x5b, 0x7a, 0x67, 0x66, 0x5c, 0xbb, 0xf1, 0x40, 0x76, 0xf7, + 0x59, 0xe5, 0xea, 0x59, 0x85, 0x84, 0x63, 0xa2, 0xe8, 0x35, 0x28, 0x7a, 0x5b, 0x61, 0xb7, 0x3c, + 0xa7, 0x09, 0x3b, 0x3c, 0x1e, 0xc1, 0x60, 0x7d, 0xb5, 0x86, 0x69, 0x45, 0x74, 0x1d, 0x8a, 0xc1, + 0x66, 0x43, 0x88, 0x25, 0x33, 0x37, 0x29, 0x5e, 0x2a, 0xe7, 0xf4, 0x8a, 0x51, 0xc2, 0x4b, 0x65, + 0x4c, 0x49, 0xa0, 0x2a, 0x0c, 0x32, 0xa7, 0x1c, 0x71, 0xc9, 0x65, 0xb2, 0xf3, 0x5d, 0x9c, 0xdb, + 0x78, 0x98, 0x03, 0x86, 0x80, 0x39, 0x21, 0xb4, 0x01, 0x43, 0x75, 0x96, 0x13, 0x53, 0x44, 0x7c, + 0xfb, 0x50, 0xa6, 0x00, 0xb2, 0x4b, 0xb2, 0x50, 0x21, 0x8f, 0x63, 0x18, 0x58, 0xd0, 0x62, 0x54, + 0x49, 0x7b, 0x67, 0x2b, 0x14, 0x39, 0x9c, 0xb3, 0xa9, 0x76, 0xc9, 0x81, 0x2b, 0xa8, 0x32, 0x0c, + 0x2c, 0x68, 0xa1, 0x57, 0xa0, 0xb0, 0x55, 0x17, 0x0e, 0x37, 0x99, 0x92, 0x48, 0x33, 0x08, 0xc5, + 0xd2, 0xd0, 0x83, 0x83, 0xf9, 0xc2, 0xea, 0x32, 0x2e, 0x6c, 0xd5, 0xd1, 0x3a, 0x0c, 0x6f, 0x71, + 0xb7, 0x75, 0x21, 0x6c, 0x7c, 0x32, 0xdb, 0xa3, 0x3e, 0xe5, 0xd9, 0xce, 0x7d, 0x4d, 0x04, 0x00, + 0x4b, 0x22, 0x2c, 0x10, 0xbe, 0x72, 0xbf, 0x17, 0xc1, 0xd1, 0x16, 0x8e, 0x16, 0x32, 0x81, 0x33, + 0x1d, 0xb1, 0x13, 0x3f, 0xd6, 0x28, 0xd2, 0x55, 0xed, 0xc8, 0x44, 0xfa, 0x22, 0x4c, 0x4c, 0xe6, + 0xaa, 0x56, 0xd9, 0xf6, 0xbb, 0xad, 0x6a, 0x85, 0x84, 0x63, 0xa2, 0x68, 0x17, 0xc6, 0xf7, 0xc2, + 0xf6, 0x0e, 0x91, 0x5b, 0x9a, 0x45, 0x8d, 0xc9, 0xb9, 0x97, 0xef, 0x08, 0x44, 0x37, 0x88, 0x3a, + 0x4e, 0x33, 0x75, 0x0a, 0x31, 0x9d, 0xfe, 0x1d, 0x9d, 0x18, 0x36, 0x69, 0xd3, 0xe1, 0x7f, 0xab, + 0xe3, 0x6f, 0xee, 0x47, 0x44, 0xc4, 0x34, 0xcb, 0x1c, 0xfe, 0x37, 0x38, 0x4a, 0x7a, 0xf8, 0x05, + 0x00, 0x4b, 0x22, 0xe8, 0x8e, 0x18, 0x1e, 0x76, 0x7a, 0x4e, 0xe5, 0x07, 0x1e, 0x5d, 0x94, 0x48, + 0x39, 0x83, 0xc2, 0x4e, 0xcb, 0x98, 0x14, 0x3b, 0x25, 0xdb, 0x3b, 0x7e, 0xe4, 0x7b, 0x89, 0x13, + 0x7a, 0x3a, 0xff, 0x94, 0xac, 0x66, 0xe0, 0xa7, 0x4f, 0xc9, 0x2c, 0x2c, 0x9c, 0xd9, 0x16, 0x6a, + 0xc0, 0x44, 0xdb, 0x0f, 0xa2, 0x7b, 0x7e, 0x20, 0xd7, 0x17, 0xea, 0x22, 0x2c, 0x31, 0x30, 0x45, + 0x8b, 0x2c, 0x5c, 0xa0, 0x09, 0xc1, 0x09, 0x9a, 0xe8, 0x13, 0x30, 0x1c, 0xd6, 0x9d, 0x26, 0xa9, + 0xdc, 0x9a, 0x9d, 0xc9, 0xbf, 0x7e, 0x6a, 0x1c, 0x25, 0x67, 0x75, 0xf1, 0xa8, 0xf9, 0x1c, 0x05, + 0x4b, 0x72, 0x68, 0x15, 0x06, 0x59, 0xa2, 0x33, 0x16, 0x80, 0x2f, 0x27, 0x7e, 0x6a, 0xca, 0x2a, + 0x9a, 0x9f, 0x4d, 0xac, 0x18, 0xf3, 0xea, 0x74, 0x0f, 0x88, 0x37, 0x83, 0x1f, 0xce, 0x9e, 0xce, + 0xdf, 0x03, 0xe2, 0xa9, 0x71, 0xab, 0xd6, 0x6d, 0x0f, 0x28, 0x24, 0x1c, 0x13, 0xa5, 0x27, 0x33, + 0x3d, 0x4d, 0xcf, 0x74, 0x31, 0xe7, 0xc9, 0x3d, 0x4b, 0xd9, 0xc9, 0x4c, 0x4f, 0x52, 0x4a, 0xc2, + 0xfe, 0xfd, 0xe1, 0x34, 0xcf, 0xc2, 0x5e, 0x99, 0xdf, 0x65, 0xa5, 0x14, 0x90, 0x1f, 0xee, 0x57, + 0xe8, 0x75, 0x8c, 0x2c, 0xf8, 0x17, 0x2c, 0x38, 0xd3, 0xce, 0xfc, 0x10, 0xc1, 0x00, 0xf4, 0x27, + 0x3b, 0xe3, 0x9f, 0xae, 0x82, 0x35, 0x66, 0xc3, 0x71, 0x4e, 0x4b, 0xc9, 0x67, 0x4e, 0xf1, 0x1d, + 0x3f, 0x73, 0xd6, 0x60, 0x84, 0x31, 0x99, 0x3d, 0x72, 0x44, 0x27, 0x5f, 0x7b, 0x8c, 0x95, 0x58, + 0x16, 0x15, 0xb1, 0x22, 0x81, 0x7e, 0xd0, 0x82, 0xf3, 0xc9, 0xae, 0x63, 0xc2, 0xc0, 0x22, 0xc2, + 0x23, 0x7f, 0xe0, 0xae, 0x8a, 0xef, 0x4f, 0xf1, 0xff, 0x06, 0xf2, 0x61, 0x2f, 0x04, 0xdc, 0xbd, + 0x31, 0x54, 0xce, 0x78, 0x61, 0x0f, 0x99, 0x5a, 0x85, 0x3e, 0x5e, 0xd9, 0x2f, 0xc2, 0x58, 0xcb, + 0xef, 0x78, 0x91, 0xb0, 0xfe, 0x11, 0x96, 0x08, 0x4c, 0x03, 0xbf, 0xa6, 0x95, 0x63, 0x03, 0x2b, + 0xf1, 0x36, 0x1f, 0x79, 0xe8, 0xb7, 0xf9, 0x9b, 0x30, 0xe6, 0x69, 0xe6, 0xaa, 0x82, 0x1f, 0xb8, + 0x9c, 0x1f, 0x9d, 0x55, 0x37, 0x6e, 0xe5, 0xbd, 0xd4, 0x4b, 0xb0, 0x41, 0xed, 0x64, 0x1f, 0x7c, + 0x5f, 0xb6, 0x32, 0x98, 0x7a, 0x2e, 0x02, 0xf8, 0xa8, 0x29, 0x02, 0xb8, 0x9c, 0x14, 0x01, 0xa4, + 0x24, 0xca, 0xc6, 0xeb, 0xbf, 0xff, 0xe4, 0x33, 0xfd, 0x86, 0x30, 0xb4, 0x9b, 0x70, 0xb1, 0xd7, + 0xb5, 0xc4, 0xcc, 0xc0, 0x1a, 0x4a, 0x7f, 0x18, 0x9b, 0x81, 0x35, 0x2a, 0x65, 0xcc, 0x20, 0xfd, + 0x06, 0xc7, 0xb1, 0xff, 0xab, 0x05, 0xc5, 0xaa, 0xdf, 0x38, 0x81, 0x07, 0xef, 0xc7, 0x8c, 0x07, + 0xef, 0xe3, 0xd9, 0x17, 0x62, 0x23, 0x57, 0x1e, 0xbe, 0x92, 0x90, 0x87, 0x9f, 0xcf, 0x23, 0xd0, + 0x5d, 0xfa, 0xfd, 0x93, 0x45, 0x18, 0xad, 0xfa, 0x0d, 0x65, 0x83, 0xfd, 0x1b, 0x0f, 0x63, 0x83, + 0x9d, 0x9b, 0x42, 0x41, 0xa3, 0xcc, 0xac, 0xc7, 0xa4, 0xe3, 0xe8, 0x37, 0x98, 0x29, 0xf6, 0x5d, + 0xe2, 0x6e, 0xef, 0x44, 0xa4, 0x91, 0xfc, 0x9c, 0x93, 0x33, 0xc5, 0xfe, 0x2f, 0x16, 0x4c, 0x26, + 0x5a, 0x47, 0x4d, 0x18, 0x6f, 0xea, 0xd2, 0x56, 0xb1, 0x4e, 0x1f, 0x4a, 0x50, 0x2b, 0x4c, 0x59, + 0xb5, 0x22, 0x6c, 0x12, 0x47, 0x0b, 0x00, 0x4a, 0xfd, 0x28, 0xc5, 0x7a, 0x8c, 0xeb, 0x57, 0xfa, + 0xc9, 0x10, 0x6b, 0x18, 0xe8, 0x25, 0x18, 0x8d, 0xfc, 0xb6, 0xdf, 0xf4, 0xb7, 0xf7, 0x6f, 0x10, + 0x19, 0x8e, 0x49, 0x19, 0xa8, 0x6d, 0xc4, 0x20, 0xac, 0xe3, 0xd9, 0x3f, 0x5d, 0xe4, 0x1f, 0xea, + 0x45, 0xee, 0x7b, 0x6b, 0xf2, 0xdd, 0xbd, 0x26, 0xbf, 0x6a, 0xc1, 0x14, 0x6d, 0x9d, 0xd9, 0xc0, + 0xc8, 0xcb, 0x56, 0x45, 0x65, 0xb6, 0xba, 0x44, 0x65, 0xbe, 0x4c, 0xcf, 0xae, 0x86, 0xdf, 0x89, 0x84, 0x04, 0x4d, 0x3b, 0x9c, 0x68, 0x29, 0x16, 0x50, 0x81, 0x47, 0x82, 0x40, 0xf8, 0xed, 0xe9, - 0x78, 0x24, 0x08, 0xb0, 0x80, 0xca, 0xa0, 0xcd, 0x03, 0xe9, 0x41, 0x9b, 0x79, 0x70, 0x49, 0x61, - 0x2d, 0x21, 0xd8, 0x1e, 0x2d, 0xb8, 0xa4, 0x34, 0xa3, 0x88, 0x71, 0xec, 0x9f, 0xcb, 0xc3, 0x68, - 0xc5, 0xaf, 0xc7, 0x0a, 0xc0, 0xe7, 0x0d, 0x05, 0xe0, 0xc5, 0x84, 0x02, 0x70, 0x52, 0xc7, 0xfd, - 0x40, 0xdd, 0xf7, 0xf5, 0x52, 0xf7, 0xfd, 0x13, 0x8b, 0xcd, 0x5a, 0x69, 0xad, 0xca, 0x4d, 0xaa, - 0xd0, 0x33, 0x30, 0xc2, 0x0e, 0x24, 0xe6, 0x28, 0x2a, 0xb5, 0x62, 0x2c, 0x19, 0xd1, 0x5a, 0x5c, - 0x8c, 0x75, 0x1c, 0x74, 0x05, 0x0a, 0x21, 0x71, 0x82, 0xda, 0xb6, 0x3a, 0xe3, 0x84, 0x0a, 0x8b, - 0x97, 0x61, 0x05, 0x45, 0x6f, 0xc4, 0x71, 0x0d, 0xf3, 0xd9, 0x8e, 0x67, 0x7a, 0x7f, 0xf8, 0x16, - 0xc9, 0x0e, 0x66, 0x68, 0xdf, 0x05, 0xd4, 0x89, 0xdf, 0x47, 0x40, 0xaf, 0x39, 0x33, 0xa0, 0x57, - 0xb1, 0x23, 0x98, 0xd7, 0x9f, 0x5b, 0x30, 0x5e, 0xf1, 0xeb, 0x74, 0xeb, 0x7e, 0x33, 0xed, 0x53, - 0x3d, 0xa8, 0xeb, 0x50, 0x97, 0xa0, 0xae, 0x97, 0x60, 0xb0, 0xe2, 0xd7, 0xcb, 0x95, 0x6e, 0x0e, - 0xdb, 0xf6, 0xdf, 0xb2, 0x60, 0xb8, 0xe2, 0xd7, 0x4f, 0x40, 0x38, 0xff, 0xaa, 0x29, 0x9c, 0x7f, - 0x24, 0x63, 0xdd, 0x64, 0xc8, 0xe3, 0xff, 0xc6, 0x00, 0x8c, 0xd1, 0x7e, 0xfa, 0x5b, 0x72, 0x2a, - 0x8d, 0x61, 0xb3, 0xfa, 0x18, 0x36, 0xca, 0x0b, 0xfb, 0x8d, 0x86, 0x7f, 0x2f, 0x39, 0xad, 0x2b, - 0xac, 0x14, 0x0b, 0x28, 0x7a, 0x0a, 0x0a, 0xad, 0x80, 0xec, 0xba, 0xbe, 0x60, 0x32, 0x35, 0x55, - 0x47, 0x45, 0x94, 0x63, 0x85, 0x41, 0x1f, 0x67, 0xa1, 0xeb, 0xd5, 0x48, 0x95, 0xd4, 0x7c, 0xaf, - 0xce, 0xe5, 0xd7, 0x79, 0x91, 0x98, 0x41, 0x2b, 0xc7, 0x06, 0x16, 0xba, 0x0b, 0x45, 0xf6, 0x9f, - 0x1d, 0x3b, 0x87, 0x4f, 0xf1, 0x29, 0x52, 0xbe, 0x09, 0x02, 0x38, 0xa6, 0x85, 0x9e, 0x05, 0x88, - 0x64, 0xf4, 0xee, 0x50, 0x04, 0x6f, 0x52, 0x0c, 0xb9, 0x8a, 0xeb, 0x1d, 0x62, 0x0d, 0x0b, 0x3d, - 0x09, 0xc5, 0xc8, 0x71, 0x1b, 0x37, 0x5d, 0x8f, 0x84, 0x4c, 0x2e, 0x9d, 0x97, 0x99, 0xd7, 0x44, - 0x21, 0x8e, 0xe1, 0x94, 0x21, 0x62, 0x91, 0x0d, 0x78, 0x82, 0xe0, 0x02, 0xc3, 0x66, 0x0c, 0xd1, - 0x4d, 0x55, 0x8a, 0x35, 0x0c, 0xb4, 0x0d, 0xe7, 0x5c, 0x8f, 0x25, 0x31, 0x20, 0xd5, 0x1d, 0xb7, - 0xb5, 0x7e, 0xb3, 0x7a, 0x87, 0x04, 0xee, 0xe6, 0xde, 0xa2, 0x53, 0xdb, 0x21, 0x9e, 0x4c, 0xde, - 0xf8, 0x61, 0xd1, 0xc5, 0x73, 0xe5, 0x2e, 0xb8, 0xb8, 0x2b, 0x25, 0xfb, 0x25, 0x38, 0x5d, 0xf1, - 0xeb, 0x15, 0x3f, 0x88, 0x56, 0xfc, 0xe0, 0x9e, 0x13, 0xd4, 0xe5, 0x4a, 0x99, 0x93, 0x79, 0x5e, - 0xe8, 0x51, 0x38, 0xc8, 0x0f, 0x0a, 0x23, 0xdb, 0xd8, 0x73, 0x8c, 0xf9, 0x3a, 0xa4, 0x87, 0x4d, - 0x8d, 0xb1, 0x01, 0x2a, 0xa3, 0xc7, 0x35, 0x27, 0x22, 0xe8, 0x16, 0xcb, 0x54, 0x1c, 0xdf, 0x88, - 0xa2, 0xfa, 0x13, 0x5a, 0xa6, 0xe2, 0x18, 0x98, 0x7a, 0x85, 0x9a, 0xf5, 0xed, 0xff, 0x3e, 0xc8, - 0x0e, 0xc7, 0x44, 0x56, 0x08, 0xf4, 0x59, 0x18, 0x0f, 0xc9, 0x4d, 0xd7, 0x6b, 0xdf, 0x97, 0x32, - 0x81, 0x2e, 0x3e, 0x52, 0xd5, 0x65, 0x1d, 0x93, 0x4b, 0x16, 0xcd, 0x32, 0x9c, 0xa0, 0x86, 0x9a, - 0x30, 0x7e, 0xcf, 0xf5, 0xea, 0xfe, 0xbd, 0x50, 0xd2, 0x2f, 0x64, 0x0b, 0x18, 0xef, 0x72, 0xcc, - 0x44, 0x1f, 0x8d, 0xe6, 0xee, 0x1a, 0xc4, 0x70, 0x82, 0x38, 0x5d, 0x80, 0x41, 0xdb, 0x5b, 0x08, - 0x6f, 0x87, 0x24, 0x10, 0x39, 0xa7, 0xd9, 0x02, 0xc4, 0xb2, 0x10, 0xc7, 0x70, 0xba, 0x00, 0xd9, - 0x9f, 0x6b, 0x81, 0xdf, 0xe6, 0x31, 0xf6, 0xc5, 0x02, 0xc4, 0xaa, 0x14, 0x6b, 0x18, 0x74, 0x83, - 0xb2, 0x7f, 0x6b, 0xbe, 0x87, 0x7d, 0x3f, 0x92, 0x5b, 0x9a, 0x65, 0x39, 0xd5, 0xca, 0xb1, 0x81, - 0x85, 0x56, 0x00, 0x85, 0xed, 0x56, 0xab, 0xc1, 0x8c, 0x2f, 0x9c, 0x06, 0x23, 0xc5, 0x15, 0xdf, - 0x79, 0x1e, 0x7a, 0xb4, 0xda, 0x01, 0xc5, 0x29, 0x35, 0xe8, 0x59, 0xbd, 0x29, 0xba, 0x3a, 0xc8, - 0xba, 0xca, 0x95, 0x11, 0x55, 0xde, 0x4f, 0x09, 0x43, 0xcb, 0x30, 0x1c, 0xee, 0x85, 0xb5, 0x48, - 0xc4, 0x50, 0xcb, 0x48, 0xfc, 0x53, 0x65, 0x28, 0x5a, 0xde, 0x39, 0x5e, 0x05, 0xcb, 0xba, 0xa8, - 0x06, 0xd3, 0x82, 0xe2, 0xd2, 0xb6, 0xe3, 0xa9, 0x34, 0x2a, 0xdc, 0x06, 0xf5, 0x99, 0x07, 0xfb, - 0x73, 0xd3, 0xa2, 0x65, 0x1d, 0x7c, 0xb0, 0x3f, 0x77, 0xa6, 0xe2, 0xd7, 0x53, 0x20, 0x38, 0x8d, - 0x1a, 0x5f, 0x7c, 0xb5, 0x9a, 0xdf, 0x6c, 0x55, 0x02, 0x7f, 0xd3, 0x6d, 0x90, 0x6e, 0x0a, 0x9d, - 0xaa, 0x81, 0x29, 0x16, 0x9f, 0x51, 0x86, 0x13, 0xd4, 0xec, 0x6f, 0x67, 0xfc, 0x0c, 0x4b, 0xb3, - 0x1c, 0xb5, 0x03, 0x82, 0x9a, 0x30, 0xd6, 0x62, 0xdb, 0x44, 0x44, 0xbe, 0x17, 0x6b, 0xfd, 0xf9, - 0x3e, 0x05, 0x13, 0xf7, 0xe8, 0x35, 0xa0, 0x04, 0x87, 0xec, 0xc5, 0x57, 0xd1, 0xc9, 0x61, 0x93, - 0xba, 0xfd, 0x63, 0x8f, 0xb0, 0x1b, 0xb1, 0xca, 0xa5, 0x0d, 0xc3, 0xc2, 0xe4, 0x5d, 0x3c, 0xad, - 0x66, 0xb3, 0xc5, 0x5e, 0xf1, 0xb4, 0x08, 0xb3, 0x79, 0x2c, 0xeb, 0xa2, 0xcf, 0xc0, 0x38, 0x7d, - 0xa9, 0x68, 0xf9, 0x4b, 0x4e, 0x65, 0x87, 0x26, 0x88, 0xd3, 0x96, 0x68, 0x59, 0x31, 0xf4, 0xca, - 0x38, 0x41, 0x0c, 0xbd, 0xc1, 0x8c, 0x33, 0xcc, 0xd4, 0x28, 0x3d, 0x48, 0xeb, 0x76, 0x18, 0x92, - 0xac, 0x46, 0x24, 0x2b, 0xed, 0x8a, 0x7d, 0xbc, 0x69, 0x57, 0xd0, 0x4d, 0x18, 0x13, 0xb9, 0x86, - 0xc5, 0xca, 0xcd, 0x1b, 0xd2, 0xb8, 0x31, 0xac, 0x03, 0x0f, 0x92, 0x05, 0xd8, 0xac, 0x8c, 0xb6, - 0xe0, 0xbc, 0x96, 0xfb, 0xe7, 0x5a, 0xe0, 0x30, 0x95, 0xba, 0xcb, 0x8e, 0x53, 0xed, 0xae, 0x7e, - 0xec, 0xc1, 0xfe, 0xdc, 0xf9, 0xf5, 0x6e, 0x88, 0xb8, 0x3b, 0x1d, 0x74, 0x0b, 0x4e, 0x73, 0xc7, - 0xda, 0x12, 0x71, 0xea, 0x0d, 0xd7, 0x53, 0xcc, 0x00, 0xdf, 0xf2, 0x67, 0x1f, 0xec, 0xcf, 0x9d, - 0x5e, 0x48, 0x43, 0xc0, 0xe9, 0xf5, 0xd0, 0xab, 0x50, 0xac, 0x7b, 0xa1, 0x18, 0x83, 0x21, 0x23, - 0xbd, 0x52, 0xb1, 0xb4, 0x56, 0x55, 0xdf, 0x1f, 0xff, 0xc1, 0x71, 0x05, 0xb4, 0xc5, 0x25, 0xb6, - 0x4a, 0x40, 0x32, 0xdc, 0x11, 0x12, 0x28, 0x29, 0x6a, 0x33, 0x5c, 0xeb, 0xb8, 0xaa, 0x42, 0x59, - 0x9c, 0x1b, 0x5e, 0x77, 0x06, 0x61, 0xf4, 0x3a, 0x20, 0xfa, 0x82, 0x70, 0x6b, 0x64, 0xa1, 0xc6, - 0xd2, 0x2a, 0x30, 0x01, 0x77, 0xc1, 0x74, 0xf6, 0xaa, 0x76, 0x60, 0xe0, 0x94, 0x5a, 0xe8, 0x3a, - 0x3d, 0x55, 0xf4, 0x52, 0x71, 0x6a, 0xa9, 0x64, 0x78, 0x25, 0xd2, 0x0a, 0x48, 0xcd, 0x89, 0x48, - 0xdd, 0xa4, 0x88, 0x13, 0xf5, 0x50, 0x1d, 0xce, 0x39, 0xed, 0xc8, 0x67, 0xc2, 0x70, 0x13, 0x75, - 0xdd, 0xdf, 0x21, 0x1e, 0xd3, 0x43, 0x15, 0x16, 0x2f, 0x52, 0x6e, 0x63, 0xa1, 0x0b, 0x1e, 0xee, - 0x4a, 0x85, 0x72, 0x89, 0x2a, 0xfb, 0x2d, 0x98, 0x91, 0x8e, 0x52, 0x32, 0xe0, 0xbe, 0x00, 0x23, - 0xdb, 0x7e, 0x18, 0xad, 0x91, 0xe8, 0x9e, 0x1f, 0xec, 0x88, 0x78, 0x95, 0x71, 0x8c, 0xe3, 0x18, - 0x84, 0x75, 0x3c, 0xfa, 0x0c, 0x64, 0x56, 0x12, 0xe5, 0x12, 0x53, 0x50, 0x17, 0xe2, 0x33, 0xe6, - 0x3a, 0x2f, 0xc6, 0x12, 0x2e, 0x51, 0xcb, 0x95, 0x25, 0xa6, 0x6c, 0x4e, 0xa0, 0x96, 0x2b, 0x4b, - 0x58, 0xc2, 0xe9, 0x72, 0x0d, 0xb7, 0x9d, 0x80, 0x54, 0x02, 0xbf, 0x46, 0x42, 0x2d, 0xb2, 0xf6, - 0xa3, 0x3c, 0x1a, 0x27, 0x5d, 0xae, 0xd5, 0x34, 0x04, 0x9c, 0x5e, 0x0f, 0x91, 0xce, 0xbc, 0x57, - 0xe3, 0xd9, 0x5a, 0x82, 0x4e, 0x7e, 0xa6, 0xcf, 0xd4, 0x57, 0x1e, 0x4c, 0xaa, 0x8c, 0x5b, 0x3c, - 0xfe, 0x66, 0x38, 0x33, 0xc1, 0xd6, 0x76, 0xff, 0xc1, 0x3b, 0x95, 0xde, 0xa5, 0x9c, 0xa0, 0x84, - 0x3b, 0x68, 0x1b, 0xc1, 0xac, 0x26, 0x7b, 0xa6, 0x43, 0xbe, 0x0a, 0xc5, 0xb0, 0xbd, 0x51, 0xf7, - 0x9b, 0x8e, 0xeb, 0x31, 0x65, 0xb3, 0xf6, 0x1e, 0xa9, 0x4a, 0x00, 0x8e, 0x71, 0xd0, 0x0a, 0x14, - 0x1c, 0xa9, 0x54, 0x41, 0xd9, 0x31, 0x50, 0x94, 0x2a, 0x85, 0x87, 0x05, 0x90, 0x6a, 0x14, 0x55, - 0x17, 0xbd, 0x02, 0x63, 0xc2, 0x31, 0x54, 0x24, 0x7b, 0x9c, 0x36, 0xbd, 0x77, 0xaa, 0x3a, 0x10, - 0x9b, 0xb8, 0xe8, 0x36, 0x8c, 0x44, 0x7e, 0x83, 0xb9, 0xa0, 0x50, 0x36, 0xef, 0x4c, 0x76, 0x1c, - 0xb5, 0x75, 0x85, 0xa6, 0xcb, 0x33, 0x55, 0x55, 0xac, 0xd3, 0x41, 0xeb, 0x7c, 0xbd, 0xb3, 0x08, - 0xd3, 0x24, 0x9c, 0x79, 0x24, 0xfb, 0x4e, 0x52, 0x81, 0xa8, 0xcd, 0xed, 0x20, 0x6a, 0x62, 0x9d, - 0x0c, 0xba, 0x06, 0x53, 0xad, 0xc0, 0xf5, 0xd9, 0x9a, 0x50, 0xfa, 0xb4, 0x19, 0x33, 0xbd, 0x4d, - 0x25, 0x89, 0x80, 0x3b, 0xeb, 0x30, 0xbf, 0x5e, 0x51, 0x38, 0x73, 0x96, 0xe7, 0x83, 0xe6, 0xcf, - 0x3b, 0x5e, 0x86, 0x15, 0x14, 0xad, 0xb2, 0x93, 0x98, 0x4b, 0x26, 0x66, 0x66, 0xb3, 0xc3, 0xae, - 0xe8, 0x12, 0x0c, 0xce, 0xbc, 0xaa, 0xbf, 0x38, 0xa6, 0x80, 0xea, 0x5a, 0xe2, 0x40, 0xfa, 0x62, - 0x08, 0x67, 0xce, 0x75, 0x31, 0x55, 0x4b, 0x3c, 0x2f, 0x62, 0x86, 0xc0, 0x28, 0x0e, 0x71, 0x82, - 0x26, 0xfa, 0x24, 0x4c, 0x8a, 0x30, 0x6f, 0xf1, 0x30, 0x9d, 0x8f, 0x0d, 0x7b, 0x71, 0x02, 0x86, - 0x3b, 0xb0, 0x79, 0xe4, 0x7d, 0x67, 0xa3, 0x41, 0xc4, 0xd1, 0x77, 0xd3, 0xf5, 0x76, 0xc2, 0x99, - 0x0b, 0xec, 0x7c, 0x10, 0x91, 0xf7, 0x93, 0x50, 0x9c, 0x52, 0x03, 0xad, 0xc3, 0x64, 0x2b, 0x20, - 0xa4, 0xc9, 0x18, 0x7d, 0x71, 0x9f, 0xcd, 0x71, 0xb7, 0x76, 0xda, 0x93, 0x4a, 0x02, 0x76, 0x90, - 0x52, 0x86, 0x3b, 0x28, 0xa0, 0x7b, 0x50, 0xf0, 0x77, 0x49, 0xb0, 0x4d, 0x9c, 0xfa, 0xcc, 0xc5, - 0x2e, 0x86, 0xe6, 0xe2, 0x72, 0xbb, 0x25, 0x70, 0x13, 0x3a, 0x78, 0x59, 0xdc, 0x5b, 0x07, 0x2f, - 0x1b, 0x43, 0x3f, 0x64, 0xc1, 0x59, 0x29, 0xb6, 0xaf, 0xb6, 0xe8, 0xa8, 0x2f, 0xf9, 0x5e, 0x18, - 0x05, 0xdc, 0x11, 0xfb, 0xb1, 0x6c, 0xe7, 0xe4, 0xf5, 0x8c, 0x4a, 0x4a, 0x38, 0x7a, 0x36, 0x0b, - 0x23, 0xc4, 0xd9, 0x2d, 0xa2, 0x25, 0x98, 0x0a, 0x49, 0x24, 0x0f, 0xa3, 0x85, 0x70, 0xe5, 0x8d, - 0xd2, 0xda, 0xcc, 0x25, 0xee, 0x45, 0x4e, 0x37, 0x43, 0x35, 0x09, 0xc4, 0x9d, 0xf8, 0xb3, 0xdf, - 0x0a, 0x53, 0x1d, 0xd7, 0xff, 0x61, 0x32, 0x8a, 0xcc, 0xee, 0xc0, 0x98, 0x31, 0xc4, 0xc7, 0xaa, - 0xc3, 0xfd, 0x97, 0xc3, 0x50, 0x54, 0xfa, 0x3d, 0x74, 0xd5, 0x54, 0xdb, 0x9e, 0x4d, 0xaa, 0x6d, - 0x0b, 0xf4, 0x5d, 0xaf, 0x6b, 0x6a, 0xd7, 0x53, 0x62, 0x67, 0x65, 0x6d, 0xe8, 0xfe, 0x9d, 0xa2, - 0x35, 0x71, 0x6d, 0xbe, 0x6f, 0xfd, 0xef, 0x40, 0x57, 0x09, 0xf0, 0x35, 0x98, 0xf2, 0x7c, 0xc6, - 0x73, 0x92, 0xba, 0x64, 0x28, 0x18, 0xdf, 0x50, 0xd4, 0x83, 0x51, 0x24, 0x10, 0x70, 0x67, 0x1d, - 0xda, 0x20, 0xbf, 0xf8, 0x93, 0x22, 0x67, 0xce, 0x17, 0x60, 0x01, 0x45, 0x97, 0x60, 0xb0, 0xe5, - 0xd7, 0xcb, 0x15, 0xc1, 0x6f, 0x6a, 0xa9, 0x6e, 0xeb, 0xe5, 0x0a, 0xe6, 0x30, 0xb4, 0x00, 0x43, - 0xec, 0x47, 0x38, 0x33, 0x9a, 0x1d, 0x75, 0x80, 0xd5, 0xd0, 0xf2, 0xb5, 0xb0, 0x0a, 0x58, 0x54, + 0x78, 0x24, 0x08, 0xb0, 0x80, 0xca, 0xa0, 0xcd, 0x03, 0xd9, 0x41, 0x9b, 0x79, 0x70, 0x49, 0x61, + 0x2d, 0x21, 0xd8, 0x1e, 0x2d, 0xb8, 0xa4, 0x34, 0xa3, 0x88, 0x71, 0xec, 0x9f, 0x2b, 0xc2, 0x58, + 0xd5, 0x6f, 0xc4, 0x0a, 0xc0, 0x17, 0x0d, 0x05, 0xe0, 0xc5, 0x84, 0x02, 0x70, 0x4a, 0xc7, 0x7d, + 0x4f, 0xdd, 0xf7, 0xf5, 0x52, 0xf7, 0xfd, 0x13, 0x8b, 0xcd, 0x5a, 0x79, 0xbd, 0xc6, 0x4d, 0xaa, + 0xd0, 0x73, 0x30, 0xca, 0x0e, 0x24, 0xe6, 0x28, 0x2a, 0xb5, 0x62, 0x2c, 0x19, 0xd1, 0x7a, 0x5c, + 0x8c, 0x75, 0x1c, 0x74, 0x05, 0x46, 0x42, 0xe2, 0x04, 0xf5, 0x1d, 0x75, 0xc6, 0x09, 0x15, 0x16, + 0x2f, 0xc3, 0x0a, 0x8a, 0xde, 0x88, 0xe3, 0x1a, 0x16, 0xf3, 0x1d, 0xcf, 0xf4, 0xfe, 0xf0, 0x2d, + 0x92, 0x1f, 0xcc, 0xd0, 0xbe, 0x0b, 0x28, 0x8d, 0xdf, 0x47, 0x40, 0xaf, 0x79, 0x33, 0xa0, 0x57, + 0x29, 0x15, 0xcc, 0xeb, 0x2f, 0x2c, 0x98, 0xa8, 0xfa, 0x0d, 0xba, 0x75, 0xbf, 0x99, 0xf6, 0xa9, + 0x1e, 0xd4, 0x75, 0xa8, 0x4b, 0x50, 0xd7, 0x4b, 0x30, 0x58, 0xf5, 0x1b, 0x95, 0x6a, 0x37, 0x87, + 0x6d, 0xfb, 0x6f, 0x59, 0x30, 0x5c, 0xf5, 0x1b, 0x27, 0x20, 0x9c, 0xff, 0xa8, 0x29, 0x9c, 0x7f, + 0x2c, 0x67, 0xdd, 0xe4, 0xc8, 0xe3, 0xff, 0xc6, 0x00, 0x8c, 0xd3, 0x7e, 0xfa, 0xdb, 0x72, 0x2a, + 0x8d, 0x61, 0xb3, 0xfa, 0x18, 0x36, 0xca, 0x0b, 0xfb, 0xcd, 0xa6, 0x7f, 0x2f, 0x39, 0xad, 0xab, + 0xac, 0x14, 0x0b, 0x28, 0x7a, 0x06, 0x46, 0xda, 0x01, 0xd9, 0x73, 0x7d, 0xc1, 0x64, 0x6a, 0xaa, + 0x8e, 0xaa, 0x28, 0xc7, 0x0a, 0x83, 0x3e, 0xce, 0x42, 0xd7, 0xab, 0x93, 0x1a, 0xa9, 0xfb, 0x5e, + 0x83, 0xcb, 0xaf, 0x8b, 0x22, 0x31, 0x83, 0x56, 0x8e, 0x0d, 0x2c, 0x74, 0x17, 0x4a, 0xec, 0x3f, + 0x3b, 0x76, 0x8e, 0x9e, 0xe2, 0x53, 0xa4, 0x7c, 0x13, 0x04, 0x70, 0x4c, 0x0b, 0x3d, 0x0f, 0x10, + 0xc9, 0xe8, 0xdd, 0xa1, 0x08, 0xde, 0xa4, 0x18, 0x72, 0x15, 0xd7, 0x3b, 0xc4, 0x1a, 0x16, 0x7a, + 0x1a, 0x4a, 0x91, 0xe3, 0x36, 0x6f, 0xba, 0x1e, 0x09, 0x99, 0x5c, 0xba, 0x28, 0x33, 0xaf, 0x89, + 0x42, 0x1c, 0xc3, 0x29, 0x43, 0xc4, 0x22, 0x1b, 0xf0, 0x04, 0xc1, 0x23, 0x0c, 0x9b, 0x31, 0x44, + 0x37, 0x55, 0x29, 0xd6, 0x30, 0xd0, 0x0e, 0x9c, 0x73, 0x3d, 0x96, 0xc4, 0x80, 0xd4, 0x76, 0xdd, + 0xf6, 0xc6, 0xcd, 0xda, 0x1d, 0x12, 0xb8, 0x5b, 0xfb, 0x4b, 0x4e, 0x7d, 0x97, 0x78, 0x32, 0x79, + 0xe3, 0xfb, 0x45, 0x17, 0xcf, 0x55, 0xba, 0xe0, 0xe2, 0xae, 0x94, 0xec, 0x97, 0xe1, 0x74, 0xd5, + 0x6f, 0x54, 0xfd, 0x20, 0x5a, 0xf5, 0x83, 0x7b, 0x4e, 0xd0, 0x90, 0x2b, 0x65, 0x5e, 0xe6, 0x79, + 0xa1, 0x47, 0xe1, 0x20, 0x3f, 0x28, 0x8c, 0x6c, 0x63, 0x2f, 0x30, 0xe6, 0xeb, 0x88, 0x1e, 0x36, + 0x75, 0xc6, 0x06, 0xa8, 0x8c, 0x1e, 0xd7, 0x9c, 0x88, 0xa0, 0x5b, 0x2c, 0x53, 0x71, 0x7c, 0x23, + 0x8a, 0xea, 0x4f, 0x69, 0x99, 0x8a, 0x63, 0x60, 0xe6, 0x15, 0x6a, 0xd6, 0xb7, 0xff, 0xdb, 0x20, + 0x3b, 0x1c, 0x13, 0x59, 0x21, 0xd0, 0x67, 0x60, 0x22, 0x24, 0x37, 0x5d, 0xaf, 0x73, 0x5f, 0xca, + 0x04, 0xba, 0xf8, 0x48, 0xd5, 0x56, 0x74, 0x4c, 0x2e, 0x59, 0x34, 0xcb, 0x70, 0x82, 0x1a, 0x6a, + 0xc1, 0xc4, 0x3d, 0xd7, 0x6b, 0xf8, 0xf7, 0x42, 0x49, 0x7f, 0x24, 0x5f, 0xc0, 0x78, 0x97, 0x63, + 0x26, 0xfa, 0x68, 0x34, 0x77, 0xd7, 0x20, 0x86, 0x13, 0xc4, 0xe9, 0x02, 0x0c, 0x3a, 0xde, 0x62, + 0x78, 0x3b, 0x24, 0x81, 0xc8, 0x39, 0xcd, 0x16, 0x20, 0x96, 0x85, 0x38, 0x86, 0xd3, 0x05, 0xc8, + 0xfe, 0x5c, 0x0b, 0xfc, 0x0e, 0x8f, 0xb1, 0x2f, 0x16, 0x20, 0x56, 0xa5, 0x58, 0xc3, 0xa0, 0x1b, + 0x94, 0xfd, 0x5b, 0xf7, 0x3d, 0xec, 0xfb, 0x91, 0xdc, 0xd2, 0x2c, 0xcb, 0xa9, 0x56, 0x8e, 0x0d, + 0x2c, 0xb4, 0x0a, 0x28, 0xec, 0xb4, 0xdb, 0x4d, 0x66, 0x7c, 0xe1, 0x34, 0x19, 0x29, 0xae, 0xf8, + 0x2e, 0xf2, 0xd0, 0xa3, 0xb5, 0x14, 0x14, 0x67, 0xd4, 0xa0, 0x67, 0xf5, 0x96, 0xe8, 0xea, 0x20, + 0xeb, 0x2a, 0x57, 0x46, 0xd4, 0x78, 0x3f, 0x25, 0x0c, 0xad, 0xc0, 0x70, 0xb8, 0x1f, 0xd6, 0x23, + 0x11, 0x43, 0x2d, 0x27, 0xf1, 0x4f, 0x8d, 0xa1, 0x68, 0x79, 0xe7, 0x78, 0x15, 0x2c, 0xeb, 0xa2, + 0x3a, 0xcc, 0x08, 0x8a, 0xcb, 0x3b, 0x8e, 0xa7, 0xd2, 0xa8, 0x70, 0x1b, 0xd4, 0xe7, 0x1e, 0x1c, + 0xcc, 0xcf, 0x88, 0x96, 0x75, 0xf0, 0xe1, 0xc1, 0xfc, 0x99, 0xaa, 0xdf, 0xc8, 0x80, 0xe0, 0x2c, + 0x6a, 0x7c, 0xf1, 0xd5, 0xeb, 0x7e, 0xab, 0x5d, 0x0d, 0xfc, 0x2d, 0xb7, 0x49, 0xba, 0x29, 0x74, + 0x6a, 0x06, 0xa6, 0x58, 0x7c, 0x46, 0x19, 0x4e, 0x50, 0xb3, 0xbf, 0x9d, 0xf1, 0x33, 0x2c, 0xcd, + 0x72, 0xd4, 0x09, 0x08, 0x6a, 0xc1, 0x78, 0x9b, 0x6d, 0x13, 0x11, 0xf9, 0x5e, 0xac, 0xf5, 0x17, + 0xfb, 0x14, 0x4c, 0xdc, 0xa3, 0xd7, 0x80, 0x12, 0x1c, 0xb2, 0x17, 0x5f, 0x55, 0x27, 0x87, 0x4d, + 0xea, 0xf6, 0x8f, 0x3d, 0xc6, 0x6e, 0xc4, 0x1a, 0x97, 0x36, 0x0c, 0x0b, 0x93, 0x77, 0xf1, 0xb4, + 0x9a, 0xcb, 0x17, 0x7b, 0xc5, 0xd3, 0x22, 0xcc, 0xe6, 0xb1, 0xac, 0x8b, 0x3e, 0x0d, 0x13, 0xf4, + 0xa5, 0xa2, 0xe5, 0x2f, 0x39, 0x95, 0x1f, 0x9a, 0x20, 0x4e, 0x5b, 0xa2, 0x65, 0xc5, 0xd0, 0x2b, + 0xe3, 0x04, 0x31, 0xf4, 0x06, 0x33, 0xce, 0x30, 0x53, 0xa3, 0xf4, 0x20, 0xad, 0xdb, 0x61, 0x48, + 0xb2, 0x1a, 0x91, 0xbc, 0xb4, 0x2b, 0xf6, 0xa3, 0x4d, 0xbb, 0x82, 0x6e, 0xc2, 0xb8, 0xc8, 0x35, + 0x2c, 0x56, 0x6e, 0xd1, 0x90, 0xc6, 0x8d, 0x63, 0x1d, 0x78, 0x98, 0x2c, 0xc0, 0x66, 0x65, 0xb4, + 0x0d, 0xe7, 0xb5, 0xdc, 0x3f, 0xd7, 0x02, 0x87, 0xa9, 0xd4, 0x5d, 0x76, 0x9c, 0x6a, 0x77, 0xf5, + 0x13, 0x0f, 0x0e, 0xe6, 0xcf, 0x6f, 0x74, 0x43, 0xc4, 0xdd, 0xe9, 0xa0, 0x5b, 0x70, 0x9a, 0x3b, + 0xd6, 0x96, 0x89, 0xd3, 0x68, 0xba, 0x9e, 0x62, 0x06, 0xf8, 0x96, 0x3f, 0xfb, 0xe0, 0x60, 0xfe, + 0xf4, 0x62, 0x16, 0x02, 0xce, 0xae, 0x87, 0x3e, 0x0a, 0xa5, 0x86, 0x17, 0x8a, 0x31, 0x18, 0x32, + 0xd2, 0x2b, 0x95, 0xca, 0xeb, 0x35, 0xf5, 0xfd, 0xf1, 0x1f, 0x1c, 0x57, 0x40, 0xdb, 0x5c, 0x62, + 0xab, 0x04, 0x24, 0xc3, 0xa9, 0x90, 0x40, 0x49, 0x51, 0x9b, 0xe1, 0x5a, 0xc7, 0x55, 0x15, 0xca, + 0xe2, 0xdc, 0xf0, 0xba, 0x33, 0x08, 0xa3, 0xd7, 0x01, 0xd1, 0x17, 0x84, 0x5b, 0x27, 0x8b, 0x75, + 0x96, 0x56, 0x81, 0x09, 0xb8, 0x47, 0x4c, 0x67, 0xaf, 0x5a, 0x0a, 0x03, 0x67, 0xd4, 0x42, 0xd7, + 0xe9, 0xa9, 0xa2, 0x97, 0x8a, 0x53, 0x4b, 0x25, 0xc3, 0x2b, 0x93, 0x76, 0x40, 0xea, 0x4e, 0x44, + 0x1a, 0x26, 0x45, 0x9c, 0xa8, 0x87, 0x1a, 0x70, 0xce, 0xe9, 0x44, 0x3e, 0x13, 0x86, 0x9b, 0xa8, + 0x1b, 0xfe, 0x2e, 0xf1, 0x98, 0x1e, 0x6a, 0x64, 0xe9, 0x22, 0xe5, 0x36, 0x16, 0xbb, 0xe0, 0xe1, + 0xae, 0x54, 0x28, 0x97, 0xa8, 0xb2, 0xdf, 0x82, 0x19, 0xe9, 0x28, 0x23, 0x03, 0xee, 0x4b, 0x30, + 0xba, 0xe3, 0x87, 0xd1, 0x3a, 0x89, 0xee, 0xf9, 0xc1, 0xae, 0x88, 0x57, 0x19, 0xc7, 0x38, 0x8e, + 0x41, 0x58, 0xc7, 0xa3, 0xcf, 0x40, 0x66, 0x25, 0x51, 0x29, 0x33, 0x05, 0xf5, 0x48, 0x7c, 0xc6, + 0x5c, 0xe7, 0xc5, 0x58, 0xc2, 0x25, 0x6a, 0xa5, 0xba, 0xcc, 0x94, 0xcd, 0x09, 0xd4, 0x4a, 0x75, + 0x19, 0x4b, 0x38, 0x5d, 0xae, 0xe1, 0x8e, 0x13, 0x90, 0x6a, 0xe0, 0xd7, 0x49, 0xa8, 0x45, 0xd6, + 0x7e, 0x9c, 0x47, 0xe3, 0xa4, 0xcb, 0xb5, 0x96, 0x85, 0x80, 0xb3, 0xeb, 0x21, 0x92, 0xce, 0x7b, + 0x35, 0x91, 0xaf, 0x25, 0x48, 0xf3, 0x33, 0x7d, 0xa6, 0xbe, 0xf2, 0x60, 0x4a, 0x65, 0xdc, 0xe2, + 0xf1, 0x37, 0xc3, 0xd9, 0x49, 0xb6, 0xb6, 0xfb, 0x0f, 0xde, 0xa9, 0xf4, 0x2e, 0x95, 0x04, 0x25, + 0x9c, 0xa2, 0x6d, 0x04, 0xb3, 0x9a, 0xea, 0x99, 0x0e, 0xf9, 0x2a, 0x94, 0xc2, 0xce, 0x66, 0xc3, + 0x6f, 0x39, 0xae, 0xc7, 0x94, 0xcd, 0xda, 0x7b, 0xa4, 0x26, 0x01, 0x38, 0xc6, 0x41, 0xab, 0x30, + 0xe2, 0x48, 0xa5, 0x0a, 0xca, 0x8f, 0x81, 0xa2, 0x54, 0x29, 0x3c, 0x2c, 0x80, 0x54, 0xa3, 0xa8, + 0xba, 0xe8, 0x55, 0x18, 0x17, 0x8e, 0xa1, 0x22, 0xd9, 0xe3, 0x8c, 0xe9, 0xbd, 0x53, 0xd3, 0x81, + 0xd8, 0xc4, 0x45, 0xb7, 0x61, 0x34, 0xf2, 0x9b, 0xcc, 0x05, 0x85, 0xb2, 0x79, 0x67, 0xf2, 0xe3, + 0xa8, 0x6d, 0x28, 0x34, 0x5d, 0x9e, 0xa9, 0xaa, 0x62, 0x9d, 0x0e, 0xda, 0xe0, 0xeb, 0x9d, 0x45, + 0x98, 0x26, 0xe1, 0xec, 0x63, 0xf9, 0x77, 0x92, 0x0a, 0x44, 0x6d, 0x6e, 0x07, 0x51, 0x13, 0xeb, + 0x64, 0xd0, 0x35, 0x98, 0x6e, 0x07, 0xae, 0xcf, 0xd6, 0x84, 0xd2, 0xa7, 0xcd, 0x9a, 0xe9, 0x6d, + 0xaa, 0x49, 0x04, 0x9c, 0xae, 0xc3, 0xfc, 0x7a, 0x45, 0xe1, 0xec, 0x59, 0x9e, 0x0f, 0x9a, 0x3f, + 0xef, 0x78, 0x19, 0x56, 0x50, 0xb4, 0xc6, 0x4e, 0x62, 0x2e, 0x99, 0x98, 0x9d, 0xcb, 0x0f, 0xbb, + 0xa2, 0x4b, 0x30, 0x38, 0xf3, 0xaa, 0xfe, 0xe2, 0x98, 0x02, 0x6a, 0x68, 0x89, 0x03, 0xe9, 0x8b, + 0x21, 0x9c, 0x3d, 0xd7, 0xc5, 0x54, 0x2d, 0xf1, 0xbc, 0x88, 0x19, 0x02, 0xa3, 0x38, 0xc4, 0x09, + 0x9a, 0xe8, 0xe3, 0x30, 0x25, 0xc2, 0xbc, 0xc5, 0xc3, 0x74, 0x3e, 0x36, 0xec, 0xc5, 0x09, 0x18, + 0x4e, 0x61, 0xf3, 0xc8, 0xfb, 0xce, 0x66, 0x93, 0x88, 0xa3, 0xef, 0xa6, 0xeb, 0xed, 0x86, 0xb3, + 0x17, 0xd8, 0xf9, 0x20, 0x22, 0xef, 0x27, 0xa1, 0x38, 0xa3, 0x06, 0xda, 0x80, 0xa9, 0x76, 0x40, + 0x48, 0x8b, 0x31, 0xfa, 0xe2, 0x3e, 0x9b, 0xe7, 0x6e, 0xed, 0xb4, 0x27, 0xd5, 0x04, 0xec, 0x30, + 0xa3, 0x0c, 0xa7, 0x28, 0xa0, 0x7b, 0x30, 0xe2, 0xef, 0x91, 0x60, 0x87, 0x38, 0x8d, 0xd9, 0x8b, + 0x5d, 0x0c, 0xcd, 0xc5, 0xe5, 0x76, 0x4b, 0xe0, 0x26, 0x74, 0xf0, 0xb2, 0xb8, 0xb7, 0x0e, 0x5e, + 0x36, 0x86, 0x7e, 0xc8, 0x82, 0xb3, 0x52, 0x6c, 0x5f, 0x6b, 0xd3, 0x51, 0x5f, 0xf6, 0xbd, 0x30, + 0x0a, 0xb8, 0x23, 0xf6, 0x13, 0xf9, 0xce, 0xc9, 0x1b, 0x39, 0x95, 0x94, 0x70, 0xf4, 0x6c, 0x1e, + 0x46, 0x88, 0xf3, 0x5b, 0x44, 0xcb, 0x30, 0x1d, 0x92, 0x48, 0x1e, 0x46, 0x8b, 0xe1, 0xea, 0x1b, + 0xe5, 0xf5, 0xd9, 0x4b, 0xdc, 0x8b, 0x9c, 0x6e, 0x86, 0x5a, 0x12, 0x88, 0xd3, 0xf8, 0x73, 0xdf, + 0x0a, 0xd3, 0xa9, 0xeb, 0xff, 0x28, 0x19, 0x45, 0xe6, 0x76, 0x61, 0xdc, 0x18, 0xe2, 0x47, 0xaa, + 0xc3, 0xfd, 0x17, 0xc3, 0x50, 0x52, 0xfa, 0x3d, 0x74, 0xd5, 0x54, 0xdb, 0x9e, 0x4d, 0xaa, 0x6d, + 0x47, 0xe8, 0xbb, 0x5e, 0xd7, 0xd4, 0x6e, 0x64, 0xc4, 0xce, 0xca, 0xdb, 0xd0, 0xfd, 0x3b, 0x45, + 0x6b, 0xe2, 0xda, 0x62, 0xdf, 0xfa, 0xdf, 0x81, 0xae, 0x12, 0xe0, 0x6b, 0x30, 0xed, 0xf9, 0x8c, + 0xe7, 0x24, 0x0d, 0xc9, 0x50, 0x30, 0xbe, 0xa1, 0xa4, 0x07, 0xa3, 0x48, 0x20, 0xe0, 0x74, 0x1d, + 0xda, 0x20, 0xbf, 0xf8, 0x93, 0x22, 0x67, 0xce, 0x17, 0x60, 0x01, 0x45, 0x97, 0x60, 0xb0, 0xed, + 0x37, 0x2a, 0x55, 0xc1, 0x6f, 0x6a, 0xa9, 0x6e, 0x1b, 0x95, 0x2a, 0xe6, 0x30, 0xb4, 0x08, 0x43, + 0xec, 0x47, 0x38, 0x3b, 0x96, 0x1f, 0x75, 0x80, 0xd5, 0xd0, 0xf2, 0xb5, 0xb0, 0x0a, 0x58, 0x54, 0x64, 0xa2, 0x2f, 0xca, 0xa4, 0x33, 0xd1, 0xd7, 0xf0, 0x43, 0x8a, 0xbe, 0x24, 0x01, 0x1c, 0xd3, 0x42, 0xf7, 0xe1, 0xb4, 0xf1, 0x30, 0xe2, 0x4b, 0x84, 0x84, 0xc2, 0xf3, 0xf9, 0x52, 0xd7, 0x17, - 0x91, 0xd0, 0x17, 0x9f, 0x17, 0x9d, 0x3e, 0x5d, 0x4e, 0xa3, 0x84, 0xd3, 0x1b, 0x40, 0x0d, 0x98, - 0xaa, 0x75, 0xb4, 0x5a, 0xe8, 0xbf, 0x55, 0x35, 0xa1, 0x9d, 0x2d, 0x76, 0x12, 0x46, 0xaf, 0x40, - 0xe1, 0x1d, 0x3f, 0x64, 0x67, 0xb5, 0xe0, 0x91, 0xa5, 0xdb, 0x6c, 0xe1, 0x8d, 0x5b, 0x55, 0x56, - 0x7e, 0xb0, 0x3f, 0x37, 0x52, 0xf1, 0xeb, 0xf2, 0x2f, 0x56, 0x15, 0xd0, 0xf7, 0x5a, 0x30, 0xdb, - 0xf9, 0xf2, 0x52, 0x9d, 0x1e, 0xeb, 0xbf, 0xd3, 0xb6, 0x68, 0x74, 0x76, 0x39, 0x93, 0x1c, 0xee, - 0xd2, 0x94, 0xfd, 0xcb, 0x5c, 0xb7, 0x2b, 0x34, 0x40, 0x24, 0x6c, 0x37, 0x4e, 0x22, 0x4d, 0xe5, - 0xb2, 0xa1, 0x9c, 0x7a, 0x68, 0xfb, 0x81, 0x7f, 0x66, 0x31, 0xfb, 0x81, 0x13, 0x74, 0x14, 0x78, - 0x03, 0x0a, 0x91, 0x4c, 0x36, 0xda, 0x25, 0xb3, 0xa6, 0xd6, 0x29, 0x66, 0x43, 0xa1, 0x38, 0x56, - 0x95, 0x57, 0x54, 0x91, 0xb1, 0xff, 0x21, 0x9f, 0x01, 0x09, 0x39, 0x01, 0x1d, 0x40, 0xc9, 0xd4, - 0x01, 0xcc, 0xf5, 0xf8, 0x82, 0x0c, 0x5d, 0xc0, 0x3f, 0x30, 0xfb, 0xcd, 0x24, 0x35, 0xef, 0x77, - 0xc3, 0x15, 0xfb, 0x47, 0x2c, 0x38, 0x95, 0x66, 0xe9, 0x49, 0x5f, 0x19, 0x5c, 0x4e, 0xa4, 0x0c, - 0x79, 0xd4, 0x08, 0xde, 0x11, 0xe5, 0x58, 0x61, 0xf4, 0x9d, 0xed, 0xea, 0x70, 0xd1, 0x5f, 0x6f, - 0xc1, 0x58, 0x25, 0x20, 0xda, 0x85, 0xf6, 0x1a, 0x77, 0xa3, 0xe6, 0xfd, 0x79, 0xea, 0xd0, 0x2e, - 0xd4, 0xf6, 0xcf, 0xe4, 0xe0, 0x14, 0xd7, 0xc4, 0x2f, 0xec, 0xfa, 0x6e, 0xbd, 0xe2, 0xd7, 0x45, - 0xa6, 0xb2, 0x37, 0x61, 0xb4, 0xa5, 0x09, 0xf7, 0xba, 0x45, 0x32, 0xd4, 0x85, 0x80, 0xb1, 0x38, - 0x42, 0x2f, 0xc5, 0x06, 0x2d, 0x54, 0x87, 0x51, 0xb2, 0xeb, 0xd6, 0x94, 0x3a, 0x37, 0x77, 0xe8, - 0xcb, 0x45, 0xb5, 0xb2, 0xac, 0xd1, 0xc1, 0x06, 0xd5, 0x63, 0xc8, 0x41, 0x6b, 0xff, 0xa8, 0x05, - 0x8f, 0x64, 0xc4, 0x3d, 0xa4, 0xcd, 0xdd, 0x63, 0x36, 0x0f, 0x22, 0x9d, 0xa5, 0x6a, 0x8e, 0x5b, - 0x42, 0x60, 0x01, 0x45, 0x9f, 0x02, 0xe0, 0x96, 0x0c, 0xf4, 0x99, 0xdb, 0x2b, 0x40, 0x9c, 0x11, - 0xdb, 0x4a, 0x0b, 0x53, 0x24, 0xeb, 0x63, 0x8d, 0x96, 0xfd, 0x53, 0x79, 0x18, 0xe4, 0xe9, 0xc4, - 0x57, 0x60, 0x78, 0x9b, 0xe7, 0x6f, 0xe8, 0x27, 0x55, 0x44, 0x2c, 0x80, 0xe0, 0x05, 0x58, 0x56, - 0x46, 0xab, 0x30, 0xcd, 0xf3, 0x5f, 0x34, 0x4a, 0xa4, 0xe1, 0xec, 0x49, 0x69, 0x19, 0xcf, 0x1d, - 0xa9, 0xa4, 0x86, 0xe5, 0x4e, 0x14, 0x9c, 0x56, 0x0f, 0xbd, 0x06, 0xe3, 0xf4, 0xf5, 0xe2, 0xb7, - 0x23, 0x49, 0x89, 0x67, 0xbe, 0x50, 0xcf, 0xa5, 0x75, 0x03, 0x8a, 0x13, 0xd8, 0xf4, 0x01, 0xdd, - 0xea, 0x90, 0x0b, 0x0e, 0xc6, 0x0f, 0x68, 0x53, 0x16, 0x68, 0xe2, 0x32, 0x13, 0xcf, 0x36, 0x33, - 0x68, 0x5d, 0xdf, 0x0e, 0x48, 0xb8, 0xed, 0x37, 0xea, 0x8c, 0xd1, 0x1a, 0xd4, 0x4c, 0x3c, 0x13, - 0x70, 0xdc, 0x51, 0x83, 0x52, 0xd9, 0x74, 0xdc, 0x46, 0x3b, 0x20, 0x31, 0x95, 0x21, 0x93, 0xca, - 0x4a, 0x02, 0x8e, 0x3b, 0x6a, 0xd0, 0x75, 0x74, 0xba, 0x12, 0xf8, 0xf4, 0xf0, 0x92, 0xc1, 0x5c, - 0x94, 0xdd, 0xee, 0xb0, 0xf4, 0x3b, 0xed, 0x12, 0xf6, 0x4c, 0x58, 0x36, 0x72, 0x0a, 0x86, 0xd2, - 0xbe, 0x2a, 0x3c, 0x4e, 0x25, 0x15, 0xf4, 0x0c, 0x8c, 0x88, 0xac, 0x06, 0xcc, 0xbc, 0x94, 0x4f, - 0x1d, 0x33, 0x32, 0x28, 0xc5, 0xc5, 0x58, 0xc7, 0xb1, 0xbf, 0x2f, 0x07, 0xd3, 0x29, 0xfe, 0x01, - 0xfc, 0xa8, 0xda, 0x72, 0xc3, 0x48, 0xe5, 0xc7, 0xd3, 0x8e, 0x2a, 0x5e, 0x8e, 0x15, 0x06, 0xdd, - 0x0f, 0xfc, 0x30, 0x4c, 0x1e, 0x80, 0xc2, 0xfe, 0x56, 0x40, 0x0f, 0x99, 0x69, 0xee, 0x22, 0x0c, - 0xb4, 0x43, 0x22, 0x03, 0x16, 0xaa, 0xf3, 0x9b, 0xe9, 0x9e, 0x18, 0x84, 0xb2, 0xc7, 0x5b, 0x4a, - 0x8d, 0xa3, 0xb1, 0xc7, 0x5c, 0x91, 0xc3, 0x61, 0xb4, 0x73, 0x11, 0xf1, 0x1c, 0x2f, 0x12, 0x4c, - 0x74, 0x1c, 0x79, 0x8b, 0x95, 0x62, 0x01, 0xb5, 0xbf, 0x94, 0x87, 0xb3, 0x99, 0x1e, 0x43, 0xb4, - 0xeb, 0x4d, 0xdf, 0x73, 0x23, 0x5f, 0x59, 0x6f, 0xf0, 0x68, 0x5b, 0xa4, 0xb5, 0xbd, 0x2a, 0xca, - 0xb1, 0xc2, 0x40, 0x97, 0x61, 0x90, 0x49, 0xae, 0x3a, 0x32, 0x05, 0x2e, 0x96, 0x78, 0xf8, 0x15, - 0x0e, 0xee, 0x3b, 0x0b, 0xeb, 0x25, 0x18, 0x68, 0xf9, 0x7e, 0x23, 0x79, 0x68, 0xd1, 0xee, 0xfa, - 0x7e, 0x03, 0x33, 0x20, 0xfa, 0x88, 0x18, 0xaf, 0x84, 0xb9, 0x02, 0x76, 0xea, 0x7e, 0xa8, 0x0d, - 0xda, 0x13, 0x30, 0xbc, 0x43, 0xf6, 0x02, 0xd7, 0xdb, 0x4a, 0x9a, 0xb1, 0xdc, 0xe0, 0xc5, 0x58, - 0xc2, 0xcd, 0xa4, 0x4f, 0xc3, 0x47, 0x9d, 0x3e, 0xb5, 0xd0, 0xf3, 0x0a, 0xfc, 0x81, 0x3c, 0x4c, - 0xe0, 0xc5, 0xd2, 0x07, 0x13, 0x71, 0xbb, 0x73, 0x22, 0x8e, 0x3a, 0x7d, 0x6a, 0xef, 0xd9, 0xf8, - 0x05, 0x0b, 0x26, 0x58, 0x6e, 0x05, 0x11, 0xa7, 0xc9, 0xf5, 0xbd, 0x13, 0x60, 0xf1, 0x2e, 0xc1, - 0x60, 0x40, 0x1b, 0x4d, 0xa6, 0x08, 0x64, 0x3d, 0xc1, 0x1c, 0x86, 0xce, 0xc1, 0x00, 0xeb, 0x02, - 0x9d, 0xbc, 0x51, 0x9e, 0x5d, 0xa9, 0xe4, 0x44, 0x0e, 0x66, 0xa5, 0x2c, 0xf8, 0x08, 0x26, 0xad, - 0x86, 0xcb, 0x3b, 0x1d, 0xeb, 0x15, 0xdf, 0x1f, 0xbe, 0xc4, 0xa9, 0x5d, 0x7b, 0x6f, 0xc1, 0x47, - 0xd2, 0x49, 0x76, 0x7f, 0x3e, 0xfd, 0x51, 0x0e, 0x2e, 0xa4, 0xd6, 0xeb, 0x3b, 0xf8, 0x48, 0xf7, - 0xda, 0xc7, 0x19, 0x83, 0x3f, 0x7f, 0x82, 0x46, 0x82, 0x03, 0xfd, 0x72, 0x98, 0x83, 0x7d, 0xc4, - 0x04, 0x49, 0x1d, 0xb2, 0xf7, 0x49, 0x4c, 0x90, 0xd4, 0xbe, 0x65, 0x3c, 0xff, 0xfe, 0x22, 0x97, - 0xf1, 0x2d, 0xec, 0x21, 0x78, 0x85, 0x9e, 0x33, 0x0c, 0x18, 0x0a, 0x8e, 0x79, 0x94, 0x9f, 0x31, - 0xbc, 0x0c, 0x2b, 0x28, 0x5a, 0x80, 0x89, 0xa6, 0xeb, 0xd1, 0xc3, 0x67, 0xcf, 0x64, 0xfc, 0x54, - 0xc8, 0xa6, 0x55, 0x13, 0x8c, 0x93, 0xf8, 0xc8, 0xd5, 0xe2, 0x85, 0xe4, 0xb2, 0x93, 0x6e, 0x67, - 0xf6, 0x76, 0xde, 0xd4, 0xb9, 0xaa, 0x51, 0x4c, 0x89, 0x1d, 0xb2, 0xaa, 0xbd, 0xff, 0xf3, 0xfd, - 0xbf, 0xff, 0x47, 0xd3, 0xdf, 0xfe, 0xb3, 0xaf, 0xc0, 0xd8, 0x43, 0x0b, 0x7c, 0xed, 0xaf, 0xe6, - 0xe1, 0xd1, 0x2e, 0xdb, 0x9e, 0x9f, 0xf5, 0xc6, 0x1c, 0x68, 0x67, 0x7d, 0xc7, 0x3c, 0x54, 0xe0, - 0xd4, 0x66, 0xbb, 0xd1, 0xd8, 0x63, 0x76, 0xf8, 0xa4, 0x2e, 0x31, 0x04, 0x4f, 0x79, 0x4e, 0xe6, - 0xb3, 0x5a, 0x49, 0xc1, 0xc1, 0xa9, 0x35, 0x29, 0x43, 0x4f, 0x6f, 0x92, 0x3d, 0x45, 0x2a, 0xc1, - 0xd0, 0x63, 0x1d, 0x88, 0x4d, 0x5c, 0x74, 0x0d, 0xa6, 0x9c, 0x5d, 0xc7, 0xe5, 0x41, 0x57, 0x25, - 0x01, 0xce, 0xd1, 0x2b, 0x39, 0xdd, 0x42, 0x12, 0x01, 0x77, 0xd6, 0x41, 0xaf, 0x03, 0xf2, 0x45, - 0xee, 0xff, 0x6b, 0xc4, 0x13, 0xaa, 0x31, 0x36, 0x77, 0xf9, 0xf8, 0x48, 0xb8, 0xd5, 0x81, 0x81, - 0x53, 0x6a, 0x25, 0xe2, 0x6f, 0x0c, 0x65, 0xc7, 0xdf, 0xe8, 0x7e, 0x2e, 0xf6, 0x4c, 0xff, 0xf0, - 0x9f, 0x2c, 0x7a, 0x7d, 0x71, 0x26, 0xdf, 0x0c, 0x23, 0xf7, 0x0a, 0x33, 0x6d, 0xe3, 0x32, 0x3c, - 0x2d, 0x6a, 0xc4, 0x69, 0xcd, 0xb4, 0x2d, 0x06, 0x62, 0x13, 0x97, 0x2f, 0x88, 0x30, 0x76, 0x56, - 0x34, 0x58, 0x7c, 0x11, 0xeb, 0x46, 0x61, 0xa0, 0x4f, 0xc3, 0x70, 0xdd, 0xdd, 0x75, 0x43, 0x3f, - 0x10, 0x2b, 0xfd, 0x90, 0xea, 0x82, 0xf8, 0x1c, 0x2c, 0x71, 0x32, 0x58, 0xd2, 0xb3, 0x7f, 0x20, - 0x07, 0x63, 0xb2, 0xc5, 0x37, 0xda, 0x7e, 0xe4, 0x9c, 0xc0, 0xb5, 0x7c, 0xcd, 0xb8, 0x96, 0x3f, - 0xd2, 0x2d, 0xe0, 0x0f, 0xeb, 0x52, 0xe6, 0x75, 0x7c, 0x2b, 0x71, 0x1d, 0x3f, 0xde, 0x9b, 0x54, - 0xf7, 0x6b, 0xf8, 0x1f, 0x59, 0x30, 0x65, 0xe0, 0x9f, 0xc0, 0x6d, 0xb0, 0x62, 0xde, 0x06, 0x8f, - 0xf5, 0xfc, 0x86, 0x8c, 0x5b, 0xe0, 0xbb, 0xf3, 0x89, 0xbe, 0xb3, 0xd3, 0xff, 0x1d, 0x18, 0xd8, - 0x76, 0x82, 0x7a, 0xb7, 0x00, 0xe7, 0x1d, 0x95, 0xe6, 0xaf, 0x3b, 0x81, 0xd0, 0x0d, 0x3e, 0xa5, - 0x72, 0x5e, 0x3b, 0x41, 0x6f, 0xbd, 0x20, 0x6b, 0x0a, 0xbd, 0x04, 0x43, 0x61, 0xcd, 0x6f, 0x29, - 0xcb, 0xf9, 0x8b, 0x3c, 0x1f, 0x36, 0x2d, 0x39, 0xd8, 0x9f, 0x43, 0x66, 0x73, 0xb4, 0x18, 0x0b, - 0x7c, 0xf4, 0x26, 0x8c, 0xb1, 0x5f, 0xca, 0x50, 0x27, 0x9f, 0x9d, 0x0c, 0xa9, 0xaa, 0x23, 0x72, - 0x2b, 0x36, 0xa3, 0x08, 0x9b, 0xa4, 0x66, 0xb7, 0xa0, 0xa8, 0x3e, 0xeb, 0x58, 0xf5, 0x71, 0xff, - 0x2e, 0x0f, 0xd3, 0x29, 0x6b, 0x0e, 0x85, 0xc6, 0x4c, 0x3c, 0xd3, 0xe7, 0x52, 0x7d, 0x8f, 0x73, - 0x11, 0xb2, 0xd7, 0x50, 0x5d, 0xac, 0xad, 0xbe, 0x1b, 0xbd, 0x1d, 0x92, 0x64, 0xa3, 0xb4, 0xa8, - 0x77, 0xa3, 0xb4, 0xb1, 0x13, 0x1b, 0x6a, 0xda, 0x90, 0xea, 0xe9, 0xb1, 0xce, 0xe9, 0x9f, 0xe6, - 0xe1, 0x54, 0x5a, 0x0c, 0x32, 0xf4, 0xf9, 0x44, 0x62, 0xbc, 0xe7, 0xfb, 0x8d, 0x5e, 0xc6, 0xb3, - 0xe5, 0x71, 0x19, 0xf0, 0xe2, 0xbc, 0x99, 0x2a, 0xaf, 0xe7, 0x30, 0x8b, 0x36, 0x99, 0x23, 0x7e, - 0xc0, 0x13, 0x1a, 0xca, 0xe3, 0xe3, 0xe3, 0x7d, 0x77, 0x40, 0x64, 0x42, 0x0c, 0x13, 0x46, 0x00, - 0xb2, 0xb8, 0xb7, 0x11, 0x80, 0x6c, 0x79, 0xd6, 0x85, 0x11, 0xed, 0x6b, 0x8e, 0x75, 0xc6, 0x77, - 0xe8, 0x6d, 0xa5, 0xf5, 0xfb, 0x58, 0x67, 0xfd, 0x47, 0x2d, 0x48, 0xd8, 0x85, 0x2b, 0xb1, 0x98, - 0x95, 0x29, 0x16, 0xbb, 0x08, 0x03, 0x81, 0xdf, 0x20, 0xc9, 0x3c, 0x74, 0xd8, 0x6f, 0x10, 0xcc, - 0x20, 0x14, 0x23, 0x8a, 0x85, 0x1d, 0xa3, 0xfa, 0x43, 0x4e, 0x3c, 0xd1, 0x2e, 0xc1, 0x60, 0x83, - 0xec, 0x92, 0x46, 0x32, 0x5d, 0xc8, 0x4d, 0x5a, 0x88, 0x39, 0xcc, 0xfe, 0x85, 0x01, 0x38, 0xdf, - 0x35, 0x94, 0x05, 0x7d, 0x0e, 0x6d, 0x39, 0x11, 0xb9, 0xe7, 0xec, 0x25, 0xe3, 0xfa, 0x5f, 0xe3, - 0xc5, 0x58, 0xc2, 0x99, 0xe7, 0x0e, 0x0f, 0xcf, 0x9b, 0x10, 0x22, 0x8a, 0xa8, 0xbc, 0x02, 0x6a, - 0x0a, 0xa5, 0xf2, 0x47, 0x21, 0x94, 0x7a, 0x16, 0x20, 0x0c, 0x1b, 0xdc, 0x7a, 0xa6, 0x2e, 0x5c, - 0x82, 0xe2, 0x30, 0xce, 0xd5, 0x9b, 0x02, 0x82, 0x35, 0x2c, 0x54, 0x82, 0xc9, 0x56, 0xe0, 0x47, - 0x5c, 0x26, 0x5b, 0xe2, 0x06, 0x66, 0x83, 0x66, 0x14, 0x81, 0x4a, 0x02, 0x8e, 0x3b, 0x6a, 0xa0, - 0x17, 0x60, 0x44, 0x44, 0x16, 0xa8, 0xf8, 0x7e, 0x43, 0x88, 0x81, 0x94, 0xcd, 0x55, 0x35, 0x06, - 0x61, 0x1d, 0x4f, 0xab, 0xc6, 0x04, 0xbd, 0xc3, 0xa9, 0xd5, 0xb8, 0xb0, 0x57, 0xc3, 0x4b, 0xc4, - 0x23, 0x2c, 0xf4, 0x15, 0x8f, 0x30, 0x16, 0x8c, 0x15, 0xfb, 0xd6, 0x6d, 0x41, 0x4f, 0x51, 0xd2, - 0xcf, 0x0e, 0xc0, 0xb4, 0x58, 0x38, 0xc7, 0xbd, 0x5c, 0x6e, 0x77, 0x2e, 0x97, 0xa3, 0x10, 0x9d, - 0x7d, 0xb0, 0x66, 0x4e, 0x7a, 0xcd, 0xfc, 0xa0, 0x05, 0x26, 0x7b, 0x85, 0xfe, 0xdf, 0xcc, 0xc4, - 0x28, 0x2f, 0x64, 0xb2, 0x6b, 0x75, 0x79, 0x81, 0xbc, 0xc7, 0x14, 0x29, 0xf6, 0x7f, 0xb0, 0xe0, - 0xb1, 0x9e, 0x14, 0xd1, 0x32, 0x14, 0x19, 0x0f, 0xa8, 0xbd, 0xce, 0x1e, 0x57, 0x06, 0xa8, 0x12, - 0x90, 0xc1, 0x92, 0xc6, 0x35, 0xd1, 0x72, 0x47, 0x06, 0x9a, 0x27, 0x52, 0x32, 0xd0, 0x9c, 0x36, - 0x86, 0xe7, 0x21, 0x53, 0xd0, 0x7c, 0x3f, 0xbd, 0x71, 0x0c, 0xe7, 0x0f, 0xf4, 0x71, 0x43, 0xec, - 0x67, 0x27, 0xc4, 0x7e, 0xc8, 0xc4, 0xd6, 0xee, 0x90, 0x4f, 0xc2, 0x24, 0x0b, 0x39, 0xc4, 0xcc, - 0xa1, 0x85, 0x5b, 0x4a, 0x2e, 0x36, 0x79, 0xbc, 0x99, 0x80, 0xe1, 0x0e, 0x6c, 0xfb, 0x0f, 0xf3, - 0x30, 0xc4, 0xb7, 0xdf, 0x09, 0xbc, 0x09, 0x9f, 0x84, 0xa2, 0xdb, 0x6c, 0xb6, 0x79, 0x52, 0x91, - 0x41, 0xee, 0x8b, 0x4a, 0xe7, 0xa9, 0x2c, 0x0b, 0x71, 0x0c, 0x47, 0x2b, 0x42, 0xe2, 0xdc, 0x25, - 0xaa, 0x21, 0xef, 0xf8, 0x7c, 0xc9, 0x89, 0x1c, 0xce, 0xe0, 0xa8, 0x7b, 0x36, 0x96, 0x4d, 0xa3, - 0xcf, 0x02, 0x84, 0x51, 0xe0, 0x7a, 0x5b, 0xb4, 0x4c, 0x04, 0xf1, 0xfc, 0x68, 0x17, 0x6a, 0x55, - 0x85, 0xcc, 0x69, 0xc6, 0x67, 0x8e, 0x02, 0x60, 0x8d, 0x22, 0x9a, 0x37, 0x6e, 0xfa, 0xd9, 0xc4, - 0xdc, 0x01, 0xa7, 0x1a, 0xcf, 0xd9, 0xec, 0x8b, 0x50, 0x54, 0xc4, 0x7b, 0xc9, 0x9f, 0x46, 0x75, - 0xb6, 0xe8, 0x13, 0x30, 0x91, 0xe8, 0xdb, 0xa1, 0xc4, 0x57, 0xbf, 0x68, 0xc1, 0x04, 0xef, 0xcc, - 0xb2, 0xb7, 0x2b, 0x6e, 0x83, 0x77, 0xe1, 0x54, 0x23, 0xe5, 0x54, 0x16, 0xd3, 0xdf, 0xff, 0x29, - 0xae, 0xc4, 0x55, 0x69, 0x50, 0x9c, 0xda, 0x06, 0xba, 0x42, 0x77, 0x1c, 0x3d, 0x75, 0x9d, 0x86, - 0x70, 0x4d, 0x1d, 0xe5, 0xbb, 0x8d, 0x97, 0x61, 0x05, 0xb5, 0x7f, 0xc7, 0x82, 0x29, 0xde, 0xf3, - 0x1b, 0x64, 0x4f, 0x9d, 0x4d, 0x5f, 0xcf, 0xbe, 0x8b, 0x74, 0x56, 0xb9, 0x8c, 0x74, 0x56, 0xfa, - 0xa7, 0xe5, 0xbb, 0x7e, 0xda, 0xcf, 0x58, 0x20, 0x56, 0xc8, 0x09, 0x08, 0x21, 0xbe, 0xd5, 0x14, - 0x42, 0xcc, 0x66, 0x6f, 0x82, 0x0c, 0xe9, 0xc3, 0x9f, 0x5b, 0x30, 0xc9, 0x11, 0x62, 0x6d, 0xf9, - 0xd7, 0x75, 0x1e, 0xfa, 0x49, 0x7a, 0x7b, 0x83, 0xec, 0xad, 0xfb, 0x15, 0x27, 0xda, 0x4e, 0xff, - 0x28, 0x63, 0xb2, 0x06, 0xba, 0x4e, 0x56, 0x5d, 0x6e, 0xa0, 0x43, 0x64, 0xd2, 0x3e, 0x74, 0xb6, - 0x07, 0xfb, 0x6b, 0x16, 0x20, 0xde, 0x8c, 0xc1, 0xb8, 0x51, 0x76, 0x88, 0x95, 0x6a, 0x17, 0x5d, - 0x7c, 0x34, 0x29, 0x08, 0xd6, 0xb0, 0x8e, 0x64, 0x78, 0x12, 0x26, 0x0f, 0xf9, 0xde, 0x26, 0x0f, - 0x87, 0x18, 0xd1, 0x7f, 0x35, 0x04, 0x49, 0x07, 0x18, 0x74, 0x07, 0x46, 0x6b, 0x4e, 0xcb, 0xd9, - 0x70, 0x1b, 0x6e, 0xe4, 0x92, 0xb0, 0x9b, 0xad, 0xd4, 0x92, 0x86, 0x27, 0x94, 0xd4, 0x5a, 0x09, - 0x36, 0xe8, 0xa0, 0x79, 0x80, 0x56, 0xe0, 0xee, 0xba, 0x0d, 0xb2, 0xc5, 0x64, 0x25, 0xcc, 0x19, - 0x9e, 0x1b, 0x00, 0xc9, 0x52, 0xac, 0x61, 0xa4, 0x78, 0x1b, 0xe7, 0x8f, 0xd9, 0xdb, 0x18, 0x4e, - 0xcc, 0xdb, 0x78, 0xe0, 0x50, 0xde, 0xc6, 0x85, 0x43, 0x7b, 0x1b, 0x0f, 0xf6, 0xe5, 0x6d, 0x8c, - 0xe1, 0x8c, 0xe4, 0x3d, 0xe9, 0xff, 0x15, 0xb7, 0x41, 0xc4, 0x83, 0x83, 0x7b, 0xf0, 0xcf, 0x3e, - 0xd8, 0x9f, 0x3b, 0x83, 0x53, 0x31, 0x70, 0x46, 0x4d, 0xf4, 0x29, 0x98, 0x71, 0x1a, 0x0d, 0xff, - 0x9e, 0x9a, 0xd4, 0xe5, 0xb0, 0xe6, 0x34, 0xb8, 0x12, 0x62, 0x98, 0x51, 0x3d, 0xf7, 0x60, 0x7f, - 0x6e, 0x66, 0x21, 0x03, 0x07, 0x67, 0xd6, 0x46, 0xaf, 0x42, 0xb1, 0x15, 0xf8, 0xb5, 0x55, 0xcd, - 0x4b, 0xef, 0x02, 0x1d, 0xc0, 0x8a, 0x2c, 0x3c, 0xd8, 0x9f, 0x1b, 0x53, 0x7f, 0xd8, 0x85, 0x1f, - 0x57, 0x48, 0x71, 0x1f, 0x1e, 0x39, 0x52, 0xf7, 0xe1, 0x1d, 0x98, 0xae, 0x92, 0xc0, 0x65, 0x79, - 0xb7, 0xeb, 0xf1, 0xf9, 0xb4, 0x0e, 0xc5, 0x20, 0x71, 0x22, 0xf7, 0x15, 0x69, 0x50, 0x0b, 0xbb, - 0x2f, 0x4f, 0xe0, 0x98, 0x90, 0xfd, 0xbf, 0x2c, 0x18, 0x16, 0x0e, 0x2f, 0x27, 0xc0, 0x35, 0x2e, - 0x18, 0x9a, 0x84, 0xb9, 0xf4, 0x01, 0x63, 0x9d, 0xc9, 0xd4, 0x21, 0x94, 0x13, 0x3a, 0x84, 0xc7, - 0xba, 0x11, 0xe9, 0xae, 0x3d, 0xf8, 0x6b, 0x79, 0xca, 0xbd, 0x1b, 0xae, 0x97, 0xc7, 0x3f, 0x04, - 0x6b, 0x30, 0x1c, 0x0a, 0xd7, 0xbf, 0x5c, 0xb6, 0xad, 0x7a, 0x72, 0x12, 0x63, 0x3b, 0x36, 0xe1, - 0xec, 0x27, 0x89, 0xa4, 0xfa, 0x14, 0xe6, 0x8f, 0xd1, 0xa7, 0xb0, 0x97, 0x73, 0xea, 0xc0, 0x51, - 0x38, 0xa7, 0xda, 0x5f, 0x61, 0x37, 0xa7, 0x5e, 0x7e, 0x02, 0x4c, 0xd5, 0x35, 0xf3, 0x8e, 0xb5, - 0xbb, 0xac, 0x2c, 0xd1, 0xa9, 0x0c, 0xe6, 0xea, 0xe7, 0x2d, 0x38, 0x9f, 0xf2, 0x55, 0x1a, 0xa7, - 0xf5, 0x14, 0x14, 0x9c, 0x76, 0xdd, 0x55, 0x7b, 0x59, 0xd3, 0x27, 0x2e, 0x88, 0x72, 0xac, 0x30, - 0xd0, 0x12, 0x4c, 0x91, 0xfb, 0x2d, 0x97, 0xab, 0x52, 0x75, 0x63, 0xd3, 0x3c, 0xf7, 0x92, 0x5a, - 0x4e, 0x02, 0x71, 0x27, 0xbe, 0x0a, 0x08, 0x92, 0xcf, 0x0c, 0x08, 0xf2, 0x77, 0x2d, 0x18, 0x51, - 0xce, 0x6f, 0xc7, 0x3e, 0xda, 0x9f, 0x34, 0x47, 0xfb, 0xd1, 0x2e, 0xa3, 0x9d, 0x31, 0xcc, 0xbf, - 0x9d, 0x53, 0xfd, 0xad, 0xf8, 0x41, 0xd4, 0x07, 0x07, 0xf7, 0x12, 0x14, 0x5a, 0x81, 0x1f, 0xf9, - 0x35, 0xbf, 0x21, 0x18, 0xb8, 0x73, 0x71, 0x64, 0x1c, 0x5e, 0x7e, 0xa0, 0xfd, 0xc6, 0x0a, 0x9b, - 0xf2, 0x4e, 0x4e, 0xab, 0x25, 0x01, 0xd2, 0x06, 0x8d, 0xc5, 0x8d, 0x8d, 0x8b, 0xb1, 0x8e, 0xc3, - 0x06, 0xdc, 0x0f, 0x22, 0xc1, 0x67, 0xc5, 0x03, 0xee, 0x07, 0x11, 0x66, 0x10, 0x54, 0x07, 0x88, - 0x9c, 0x60, 0x8b, 0x44, 0xb4, 0x4c, 0x04, 0xef, 0xca, 0x3e, 0x6f, 0xda, 0x91, 0xdb, 0x98, 0x77, - 0xbd, 0x28, 0x8c, 0x82, 0xf9, 0xb2, 0x17, 0xdd, 0x0a, 0xf8, 0x13, 0x52, 0x8b, 0x8e, 0xa3, 0x68, - 0x61, 0x8d, 0xae, 0x74, 0xf4, 0x66, 0x6d, 0x0c, 0x9a, 0xc6, 0x0c, 0x6b, 0xa2, 0x1c, 0x2b, 0x0c, - 0xfb, 0x45, 0x76, 0xfb, 0xb0, 0x31, 0x3d, 0x5c, 0x38, 0x99, 0x5f, 0x2e, 0xaa, 0xd9, 0x60, 0x9a, - 0xcc, 0x92, 0x1e, 0xb4, 0xa6, 0xfb, 0x61, 0x4f, 0x1b, 0xd6, 0xfd, 0xb5, 0xe2, 0xc8, 0x36, 0xe8, - 0xdb, 0x3a, 0x0c, 0x54, 0x9e, 0xee, 0x71, 0x6b, 0x1c, 0xc2, 0x24, 0x85, 0x25, 0x91, 0x60, 0x21, - 0xf6, 0xcb, 0x15, 0xb1, 0x2f, 0xb4, 0x24, 0x12, 0x02, 0x80, 0x63, 0x1c, 0x74, 0x55, 0x08, 0x08, - 0xb8, 0x9c, 0xff, 0xd1, 0x84, 0x80, 0x40, 0x7e, 0xbe, 0x26, 0xd5, 0x79, 0x06, 0x46, 0x54, 0x5e, - 0xd8, 0x0a, 0x4f, 0x37, 0x2a, 0x96, 0xcd, 0x72, 0x5c, 0x8c, 0x75, 0x1c, 0xb4, 0x0e, 0x13, 0x21, - 0x97, 0x9b, 0xa9, 0x88, 0xb5, 0x5c, 0xfe, 0xf8, 0x51, 0x69, 0xd5, 0x53, 0x35, 0xc1, 0x07, 0xac, - 0x88, 0x9f, 0x36, 0xd2, 0xb9, 0x3a, 0x49, 0x02, 0xbd, 0x06, 0xe3, 0x0d, 0xdf, 0xa9, 0x2f, 0x3a, - 0x0d, 0xc7, 0xab, 0xb1, 0xef, 0x2d, 0x98, 0xe9, 0x05, 0x6f, 0x1a, 0x50, 0x9c, 0xc0, 0xa6, 0xcc, - 0x98, 0x5e, 0x22, 0xa2, 0x2c, 0x3b, 0xde, 0x16, 0x09, 0x45, 0x96, 0x4f, 0xc6, 0x8c, 0xdd, 0xcc, - 0xc0, 0xc1, 0x99, 0xb5, 0xd1, 0x4b, 0x30, 0x2a, 0x3f, 0x5f, 0x8b, 0x45, 0x10, 0x3b, 0x32, 0x68, - 0x30, 0x6c, 0x60, 0xa2, 0x7b, 0x70, 0x5a, 0xfe, 0x5f, 0x0f, 0x9c, 0xcd, 0x4d, 0xb7, 0x26, 0x1c, - 0x74, 0xb9, 0x97, 0xe1, 0x82, 0x74, 0x85, 0x5b, 0x4e, 0x43, 0x3a, 0xd8, 0x9f, 0xbb, 0x28, 0x46, - 0x2d, 0x15, 0xce, 0x26, 0x31, 0x9d, 0x3e, 0x5a, 0x85, 0xe9, 0x6d, 0xe2, 0x34, 0xa2, 0xed, 0xa5, - 0x6d, 0x52, 0xdb, 0x91, 0x9b, 0x88, 0x45, 0x38, 0xd0, 0xcc, 0xff, 0xaf, 0x77, 0xa2, 0xe0, 0xb4, - 0x7a, 0xe8, 0x2d, 0x98, 0x69, 0xb5, 0x37, 0x1a, 0x6e, 0xb8, 0xbd, 0xe6, 0x47, 0xcc, 0xb4, 0x47, - 0xa5, 0x99, 0x15, 0xa1, 0x10, 0x54, 0x0c, 0x89, 0x4a, 0x06, 0x1e, 0xce, 0xa4, 0x80, 0xde, 0x85, - 0xd3, 0x89, 0xc5, 0x20, 0x9c, 0xc1, 0xc7, 0xb3, 0x63, 0xd6, 0x57, 0xd3, 0x2a, 0x88, 0xb8, 0x0a, - 0x69, 0x20, 0x9c, 0xde, 0x04, 0x7a, 0x1e, 0x0a, 0x6e, 0x6b, 0xc5, 0x69, 0xba, 0x8d, 0x3d, 0x16, - 0x74, 0xbf, 0xc8, 0x02, 0xd1, 0x17, 0xca, 0x15, 0x5e, 0x76, 0xa0, 0xfd, 0xc6, 0x0a, 0x93, 0x3e, - 0x41, 0xb4, 0xd0, 0xa2, 0xe1, 0xcc, 0x64, 0x6c, 0xb9, 0xac, 0xc5, 0x1f, 0x0d, 0xb1, 0x81, 0xf5, - 0xde, 0x0c, 0xc2, 0xde, 0xa1, 0x95, 0x35, 0x9e, 0x11, 0x7d, 0x0e, 0x46, 0xf5, 0x15, 0x2b, 0xee, - 0xbf, 0xcb, 0xe9, 0x2c, 0x95, 0xb6, 0xb2, 0x39, 0xc7, 0xa9, 0x56, 0xaf, 0x0e, 0xc3, 0x06, 0x45, - 0x9b, 0x40, 0xfa, 0x58, 0xa2, 0x9b, 0x50, 0xa8, 0x35, 0x5c, 0xe2, 0x45, 0xe5, 0x4a, 0xb7, 0xa8, - 0x58, 0x4b, 0x02, 0x47, 0x4c, 0x8e, 0x08, 0x28, 0xce, 0xcb, 0xb0, 0xa2, 0x60, 0xff, 0x5a, 0x0e, - 0xe6, 0x7a, 0x44, 0xa7, 0x4f, 0xe8, 0x2d, 0xac, 0xbe, 0xf4, 0x16, 0x0b, 0x32, 0x41, 0xef, 0x5a, - 0x42, 0x24, 0x92, 0x48, 0xbe, 0x1b, 0x0b, 0x46, 0x92, 0xf8, 0x7d, 0xdb, 0x91, 0xeb, 0xaa, 0x8f, - 0x81, 0x9e, 0x9e, 0x10, 0x86, 0xca, 0x73, 0xb0, 0xff, 0x77, 0x52, 0xa6, 0xfa, 0xca, 0xfe, 0x4a, - 0x0e, 0x4e, 0xab, 0x21, 0xfc, 0xe6, 0x1d, 0xb8, 0xdb, 0x9d, 0x03, 0x77, 0x04, 0xca, 0x3f, 0xfb, - 0x16, 0x0c, 0xf1, 0x30, 0x5f, 0x7d, 0xf0, 0x67, 0x97, 0xcc, 0x88, 0x98, 0x8a, 0x25, 0x30, 0xa2, - 0x62, 0x7e, 0xaf, 0x05, 0x13, 0xeb, 0x4b, 0x95, 0xaa, 0x5f, 0xdb, 0x21, 0xd1, 0x02, 0xe7, 0xa7, - 0xb1, 0xe0, 0xb5, 0xac, 0x87, 0xe4, 0xa1, 0xd2, 0xb8, 0xb3, 0x8b, 0x30, 0xb0, 0xed, 0x87, 0x51, - 0xd2, 0x32, 0xe0, 0xba, 0x1f, 0x46, 0x98, 0x41, 0xec, 0xdf, 0xb5, 0x60, 0x90, 0xa5, 0xa4, 0x97, - 0x52, 0x64, 0x2b, 0x43, 0x8a, 0xdc, 0xcf, 0x77, 0xa1, 0x17, 0x60, 0x88, 0x6c, 0x6e, 0x92, 0x5a, - 0x24, 0x66, 0x55, 0xba, 0x72, 0x0f, 0x2d, 0xb3, 0x52, 0xca, 0x60, 0xb0, 0xc6, 0xf8, 0x5f, 0x2c, - 0x90, 0xd1, 0x5d, 0x28, 0x46, 0x6e, 0x93, 0x2c, 0xd4, 0xeb, 0x42, 0xb7, 0xfa, 0x10, 0xee, 0xe8, - 0xeb, 0x92, 0x00, 0x8e, 0x69, 0xd9, 0x5f, 0xca, 0x01, 0xc4, 0xf1, 0x51, 0x7a, 0x7d, 0xe2, 0x62, - 0x87, 0xd6, 0xed, 0x72, 0x8a, 0xd6, 0x0d, 0xc5, 0x04, 0x53, 0x54, 0x6e, 0x6a, 0x98, 0xf2, 0x7d, - 0x0d, 0xd3, 0xc0, 0x61, 0x86, 0x69, 0x09, 0xa6, 0xe2, 0xf8, 0x2e, 0x66, 0x78, 0x2b, 0xf6, 0x86, - 0x5a, 0x4f, 0x02, 0x71, 0x27, 0xbe, 0x4d, 0xe0, 0xa2, 0x0a, 0x73, 0x21, 0xee, 0x1a, 0x66, 0xba, - 0xab, 0x6b, 0x31, 0x7b, 0x8c, 0x53, 0xac, 0x56, 0xcc, 0x65, 0xaa, 0x15, 0x7f, 0xc2, 0x82, 0x53, - 0xc9, 0x76, 0x98, 0x2f, 0xe5, 0x17, 0x2d, 0x38, 0xcd, 0x94, 0xab, 0xac, 0xd5, 0x4e, 0x55, 0xee, - 0xf3, 0x5d, 0x43, 0x77, 0x64, 0xf4, 0x38, 0x8e, 0x19, 0xb0, 0x9a, 0x46, 0x1a, 0xa7, 0xb7, 0x68, - 0xff, 0xfb, 0x1c, 0xcc, 0x64, 0xc5, 0xfc, 0x60, 0x96, 0xfd, 0xce, 0xfd, 0xea, 0x0e, 0xb9, 0x27, - 0xec, 0xa7, 0x63, 0xcb, 0x7e, 0x5e, 0x8c, 0x25, 0x3c, 0x19, 0x70, 0x3c, 0xd7, 0x5f, 0xc0, 0x71, - 0xb4, 0x0d, 0x53, 0xf7, 0xb6, 0x89, 0x77, 0xdb, 0x0b, 0x9d, 0xc8, 0x0d, 0x37, 0x5d, 0xa6, 0x88, - 0xe4, 0xeb, 0xe6, 0x65, 0x69, 0xe5, 0x7c, 0x37, 0x89, 0x70, 0xb0, 0x3f, 0x77, 0xde, 0x28, 0x88, - 0xbb, 0xcc, 0x0f, 0x12, 0xdc, 0x49, 0xb4, 0x33, 0x5e, 0xfb, 0xc0, 0x31, 0xc6, 0x6b, 0xb7, 0xbf, - 0x68, 0xc1, 0xd9, 0xcc, 0x24, 0x91, 0xe8, 0x0a, 0x14, 0x9c, 0x96, 0xcb, 0x65, 0xb9, 0xe2, 0x18, - 0x65, 0x32, 0x83, 0x4a, 0x99, 0x4b, 0x72, 0x15, 0x54, 0x25, 0xaf, 0xce, 0x65, 0x26, 0xaf, 0xee, - 0x99, 0x8b, 0xda, 0xfe, 0x1e, 0x0b, 0x84, 0x57, 0x62, 0x1f, 0x67, 0xf7, 0x9b, 0x32, 0xf7, 0xbf, - 0x91, 0xd3, 0xe5, 0x62, 0xb6, 0x9b, 0xa6, 0xc8, 0xe4, 0xa2, 0x78, 0x25, 0x23, 0x7f, 0x8b, 0x41, - 0xcb, 0xae, 0x83, 0x80, 0x96, 0x08, 0x93, 0x54, 0xf6, 0xee, 0xcd, 0xb3, 0x00, 0x75, 0x86, 0xab, - 0x65, 0x00, 0x57, 0x37, 0x73, 0x49, 0x41, 0xb0, 0x86, 0x65, 0xff, 0x9b, 0x1c, 0x8c, 0xc8, 0x1c, - 0x22, 0x6d, 0xaf, 0x1f, 0x79, 0xc2, 0xa1, 0x92, 0x0a, 0xb2, 0x94, 0xf9, 0x94, 0x70, 0x25, 0x16, - 0xc3, 0xc4, 0x29, 0xf3, 0x25, 0x00, 0xc7, 0x38, 0x74, 0x17, 0x85, 0xed, 0x0d, 0x86, 0x9e, 0xf0, - 0xa1, 0xab, 0xf2, 0x62, 0x2c, 0xe1, 0xe8, 0x53, 0x30, 0xc9, 0xeb, 0x05, 0x7e, 0xcb, 0xd9, 0xe2, - 0x42, 0xf2, 0x41, 0xe5, 0xfc, 0x3e, 0xb9, 0x9a, 0x80, 0x1d, 0xec, 0xcf, 0x9d, 0x4a, 0x96, 0x31, - 0xed, 0x4f, 0x07, 0x15, 0x66, 0x0b, 0xc3, 0x1b, 0xa1, 0xbb, 0xbf, 0xc3, 0x84, 0x26, 0x06, 0x61, - 0x1d, 0xcf, 0xfe, 0x1c, 0xa0, 0xce, 0x6c, 0x2a, 0xe8, 0x75, 0x6e, 0x00, 0xe9, 0x06, 0xa4, 0xde, - 0x4d, 0x1b, 0xa4, 0xbb, 0x78, 0x4b, 0xf7, 0x17, 0x5e, 0x0b, 0xab, 0xfa, 0xf6, 0x5f, 0xca, 0xc3, - 0x64, 0xd2, 0xe1, 0x17, 0x5d, 0x87, 0x21, 0xce, 0x7a, 0x08, 0xf2, 0x5d, 0x8c, 0x0d, 0x34, 0x37, - 0x61, 0x76, 0x08, 0x0b, 0xee, 0x45, 0xd4, 0x47, 0x6f, 0xc1, 0x48, 0xdd, 0xbf, 0xe7, 0xdd, 0x73, - 0x82, 0xfa, 0x42, 0xa5, 0x2c, 0x96, 0x73, 0xea, 0x6b, 0xa9, 0x14, 0xa3, 0xe9, 0xae, 0xc7, 0x4c, - 0xb1, 0x16, 0x83, 0xb0, 0x4e, 0x0e, 0xad, 0xb3, 0xe0, 0xcf, 0x9b, 0xee, 0xd6, 0xaa, 0xd3, 0xea, - 0x66, 0x0d, 0xbf, 0x24, 0x91, 0x34, 0xca, 0x63, 0x22, 0x42, 0x34, 0x07, 0xe0, 0x98, 0x10, 0xfa, - 0x3c, 0x4c, 0x87, 0x19, 0x32, 0xd9, 0xac, 0xe4, 0x5a, 0xdd, 0xc4, 0x94, 0x8b, 0x8f, 0xd0, 0x77, - 0x6c, 0x9a, 0xf4, 0x36, 0xad, 0x19, 0xfb, 0x47, 0x4e, 0x81, 0xb1, 0x89, 0x8d, 0x5c, 0x8b, 0xd6, - 0x11, 0xe5, 0x5a, 0xc4, 0x50, 0x20, 0xcd, 0x56, 0xb4, 0x57, 0x72, 0x83, 0x6e, 0x19, 0x88, 0x97, - 0x05, 0x4e, 0x27, 0x4d, 0x09, 0xc1, 0x8a, 0x4e, 0x7a, 0x42, 0xcc, 0xfc, 0xd7, 0x31, 0x21, 0xe6, - 0xc0, 0x09, 0x26, 0xc4, 0x5c, 0x83, 0xe1, 0x2d, 0x37, 0xc2, 0xa4, 0xe5, 0x0b, 0xa6, 0x3f, 0x75, - 0x1d, 0x5e, 0xe3, 0x28, 0x9d, 0xa9, 0xd7, 0x04, 0x00, 0x4b, 0x22, 0xe8, 0x75, 0xb5, 0x03, 0x87, - 0xb2, 0xdf, 0xcc, 0x9d, 0x5a, 0xf1, 0xd4, 0x3d, 0x28, 0xd2, 0x5e, 0x0e, 0x3f, 0x6c, 0xda, 0xcb, - 0x15, 0x99, 0xac, 0xb2, 0x90, 0xed, 0xba, 0xc2, 0x72, 0x51, 0xf6, 0x48, 0x51, 0x79, 0x47, 0x4f, - 0xf0, 0x59, 0xcc, 0x3e, 0x09, 0x54, 0xee, 0xce, 0x3e, 0xd3, 0x7a, 0x7e, 0x8f, 0x05, 0xa7, 0x5b, - 0x69, 0xb9, 0x6e, 0x85, 0x02, 0xf9, 0x85, 0xbe, 0xd3, 0xe9, 0x1a, 0x0d, 0x32, 0x41, 0x4d, 0x2a, - 0x1a, 0x4e, 0x6f, 0x8e, 0x0e, 0x74, 0xb0, 0x51, 0x17, 0x8a, 0xcc, 0x4b, 0x19, 0xf9, 0x41, 0xbb, - 0x64, 0x05, 0x5d, 0x4f, 0xc9, 0x45, 0xf9, 0xe1, 0xac, 0x5c, 0x94, 0x7d, 0x67, 0xa0, 0x7c, 0x5d, - 0x65, 0x06, 0x1d, 0xcb, 0x5e, 0x4a, 0x3c, 0xef, 0x67, 0xcf, 0x7c, 0xa0, 0xaf, 0xab, 0x7c, 0xa0, - 0x5d, 0x22, 0x7b, 0xf2, 0x6c, 0x9f, 0x3d, 0xb3, 0x80, 0x6a, 0x99, 0x3c, 0x27, 0x8e, 0x26, 0x93, - 0xa7, 0x71, 0xd5, 0xf0, 0x64, 0x92, 0x4f, 0xf6, 0xb8, 0x6a, 0x0c, 0xba, 0xdd, 0x2f, 0x1b, 0x9e, - 0xb5, 0x74, 0xea, 0xa1, 0xb2, 0x96, 0xde, 0xd1, 0xb3, 0x80, 0xa2, 0x1e, 0x69, 0x2e, 0x29, 0x52, - 0x9f, 0xb9, 0x3f, 0xef, 0xe8, 0x17, 0xe0, 0x74, 0x36, 0x5d, 0x75, 0xcf, 0x75, 0xd2, 0x4d, 0xbd, - 0x02, 0x3b, 0x72, 0x8a, 0x9e, 0x3a, 0x99, 0x9c, 0xa2, 0xa7, 0x8f, 0x3c, 0xa7, 0xe8, 0x99, 0x13, - 0xc8, 0x29, 0xfa, 0xc8, 0x09, 0xe6, 0x14, 0xbd, 0xc3, 0xac, 0x2e, 0x78, 0x6c, 0x17, 0x11, 0x89, - 0x34, 0x3d, 0xea, 0x65, 0x5a, 0x00, 0x18, 0xfe, 0x71, 0x0a, 0x84, 0x63, 0x52, 0x29, 0xb9, 0x4a, - 0x67, 0x8e, 0x21, 0x57, 0xe9, 0x5a, 0x9c, 0xab, 0xf4, 0x6c, 0xf6, 0x54, 0xa7, 0xd8, 0xe9, 0x67, - 0x64, 0x28, 0xbd, 0xa3, 0x67, 0x16, 0x7d, 0xb4, 0x8b, 0x28, 0x3e, 0x4d, 0xf0, 0xd8, 0x25, 0x9f, - 0xe8, 0x6b, 0x3c, 0x9f, 0xe8, 0xb9, 0xec, 0x93, 0x3c, 0x79, 0xdd, 0x19, 0x59, 0x44, 0x69, 0xbf, - 0x54, 0xcc, 0x3b, 0x16, 0x73, 0x35, 0xa3, 0x5f, 0x2a, 0x68, 0x5e, 0x67, 0xbf, 0x14, 0x08, 0xc7, - 0xa4, 0xec, 0xef, 0xcb, 0xc1, 0x85, 0xee, 0xfb, 0x2d, 0x96, 0xa6, 0x56, 0x62, 0x4d, 0x63, 0x42, - 0x9a, 0xca, 0xdf, 0x6c, 0x31, 0x56, 0xdf, 0xe1, 0xc4, 0xae, 0xc1, 0x94, 0x32, 0xf0, 0x6f, 0xb8, - 0xb5, 0xbd, 0xb5, 0xf8, 0xe5, 0xab, 0x9c, 0xa2, 0xab, 0x49, 0x04, 0xdc, 0x59, 0x07, 0x2d, 0xc0, - 0x84, 0x51, 0x58, 0x2e, 0x89, 0xb7, 0x99, 0x12, 0xdf, 0x56, 0x4d, 0x30, 0x4e, 0xe2, 0xdb, 0x5f, - 0xb6, 0xe0, 0x91, 0x8c, 0x34, 0x60, 0x7d, 0x47, 0xcb, 0xda, 0x84, 0x89, 0x96, 0x59, 0xb5, 0x47, - 0x50, 0x3d, 0x23, 0xd9, 0x98, 0xea, 0x6b, 0x02, 0x80, 0x93, 0x44, 0xed, 0x3f, 0xb3, 0xe0, 0x7c, - 0x57, 0x8b, 0x35, 0x84, 0xe1, 0xcc, 0x56, 0x33, 0x74, 0x96, 0x02, 0x52, 0x27, 0x5e, 0xe4, 0x3a, - 0x8d, 0x6a, 0x8b, 0xd4, 0x34, 0x79, 0x38, 0x33, 0xfd, 0xba, 0xb6, 0x5a, 0x5d, 0xe8, 0xc4, 0xc0, - 0x19, 0x35, 0xd1, 0x0a, 0xa0, 0x4e, 0x88, 0x98, 0x61, 0x16, 0xbd, 0xb7, 0x93, 0x1e, 0x4e, 0xa9, - 0x81, 0x5e, 0x84, 0x31, 0x65, 0x09, 0xa7, 0xcd, 0x38, 0x3b, 0xd8, 0xb1, 0x0e, 0xc0, 0x26, 0xde, - 0xe2, 0x95, 0xdf, 0xf8, 0xfd, 0x0b, 0x1f, 0xfa, 0xad, 0xdf, 0xbf, 0xf0, 0xa1, 0xdf, 0xf9, 0xfd, - 0x0b, 0x1f, 0xfa, 0x8e, 0x07, 0x17, 0xac, 0xdf, 0x78, 0x70, 0xc1, 0xfa, 0xad, 0x07, 0x17, 0xac, - 0xdf, 0x79, 0x70, 0xc1, 0xfa, 0xbd, 0x07, 0x17, 0xac, 0x2f, 0xfd, 0xc1, 0x85, 0x0f, 0xbd, 0x99, - 0xdb, 0x7d, 0xe6, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x9f, 0xac, 0x23, 0x24, 0x01, 0x01, - 0x00, + 0x91, 0xd0, 0x17, 0x9f, 0x17, 0x9d, 0x3e, 0x5d, 0xc9, 0xa2, 0x84, 0xb3, 0x1b, 0x40, 0x4d, 0x98, + 0xae, 0xa7, 0x5a, 0x1d, 0xe9, 0xbf, 0x55, 0x35, 0xa1, 0xe9, 0x16, 0xd3, 0x84, 0xd1, 0xab, 0x30, + 0xf2, 0x96, 0x1f, 0xb2, 0xb3, 0x5a, 0xf0, 0xc8, 0xd2, 0x6d, 0x76, 0xe4, 0x8d, 0x5b, 0x35, 0x56, + 0x7e, 0x78, 0x30, 0x3f, 0x5a, 0xf5, 0x1b, 0xf2, 0x2f, 0x56, 0x15, 0xd0, 0xf7, 0x5a, 0x30, 0x97, + 0x7e, 0x79, 0xa9, 0x4e, 0x8f, 0xf7, 0xdf, 0x69, 0x5b, 0x34, 0x3a, 0xb7, 0x92, 0x4b, 0x0e, 0x77, + 0x69, 0xca, 0xfe, 0x65, 0xae, 0xdb, 0x15, 0x1a, 0x20, 0x12, 0x76, 0x9a, 0x27, 0x91, 0xa6, 0x72, + 0xc5, 0x50, 0x4e, 0x3d, 0xb4, 0xfd, 0xc0, 0xaf, 0x5b, 0xcc, 0x7e, 0xe0, 0x04, 0x1d, 0x05, 0xde, + 0x80, 0x91, 0x48, 0x26, 0x1b, 0xed, 0x92, 0x59, 0x53, 0xeb, 0x14, 0xb3, 0xa1, 0x50, 0x1c, 0xab, + 0xca, 0x2b, 0xaa, 0xc8, 0xd8, 0xff, 0x90, 0xcf, 0x80, 0x84, 0x9c, 0x80, 0x0e, 0xa0, 0x6c, 0xea, + 0x00, 0xe6, 0x7b, 0x7c, 0x41, 0x8e, 0x2e, 0xe0, 0x1f, 0x98, 0xfd, 0x66, 0x92, 0x9a, 0x77, 0xbb, + 0xe1, 0x8a, 0xfd, 0x23, 0x16, 0x9c, 0xca, 0xb2, 0xf4, 0xa4, 0xaf, 0x0c, 0x2e, 0x27, 0x52, 0x86, + 0x3c, 0x6a, 0x04, 0xef, 0x88, 0x72, 0xac, 0x30, 0xfa, 0xce, 0x76, 0x75, 0xb4, 0xe8, 0xaf, 0xb7, + 0x60, 0xbc, 0x1a, 0x10, 0xed, 0x42, 0x7b, 0x8d, 0xbb, 0x51, 0xf3, 0xfe, 0x3c, 0x73, 0x64, 0x17, + 0x6a, 0xfb, 0x67, 0x0a, 0x70, 0x8a, 0x6b, 0xe2, 0x17, 0xf7, 0x7c, 0xb7, 0x51, 0xf5, 0x1b, 0x22, + 0x53, 0xd9, 0xa7, 0x60, 0xac, 0xad, 0x09, 0xf7, 0xba, 0x45, 0x32, 0xd4, 0x85, 0x80, 0xb1, 0x38, + 0x42, 0x2f, 0xc5, 0x06, 0x2d, 0xd4, 0x80, 0x31, 0xb2, 0xe7, 0xd6, 0x95, 0x3a, 0xb7, 0x70, 0xe4, + 0xcb, 0x45, 0xb5, 0xb2, 0xa2, 0xd1, 0xc1, 0x06, 0xd5, 0x47, 0x90, 0x83, 0xd6, 0xfe, 0x51, 0x0b, + 0x1e, 0xcb, 0x89, 0x7b, 0x48, 0x9b, 0xbb, 0xc7, 0x6c, 0x1e, 0x44, 0x3a, 0x4b, 0xd5, 0x1c, 0xb7, + 0x84, 0xc0, 0x02, 0x8a, 0x3e, 0x01, 0xc0, 0x2d, 0x19, 0xe8, 0x33, 0xb7, 0x57, 0x80, 0x38, 0x23, + 0xb6, 0x95, 0x16, 0xa6, 0x48, 0xd6, 0xc7, 0x1a, 0x2d, 0xfb, 0xa7, 0x8a, 0x30, 0xc8, 0xd3, 0x89, + 0xaf, 0xc2, 0xf0, 0x0e, 0xcf, 0xdf, 0xd0, 0x4f, 0xaa, 0x88, 0x58, 0x00, 0xc1, 0x0b, 0xb0, 0xac, + 0x8c, 0xd6, 0x60, 0x86, 0xe7, 0xbf, 0x68, 0x96, 0x49, 0xd3, 0xd9, 0x97, 0xd2, 0x32, 0x9e, 0x3b, + 0x52, 0x49, 0x0d, 0x2b, 0x69, 0x14, 0x9c, 0x55, 0x0f, 0xbd, 0x06, 0x13, 0xf4, 0xf5, 0xe2, 0x77, + 0x22, 0x49, 0x89, 0x67, 0xbe, 0x50, 0xcf, 0xa5, 0x0d, 0x03, 0x8a, 0x13, 0xd8, 0xf4, 0x01, 0xdd, + 0x4e, 0xc9, 0x05, 0x07, 0xe3, 0x07, 0xb4, 0x29, 0x0b, 0x34, 0x71, 0x99, 0x89, 0x67, 0x87, 0x19, + 0xb4, 0x6e, 0xec, 0x04, 0x24, 0xdc, 0xf1, 0x9b, 0x0d, 0xc6, 0x68, 0x0d, 0x6a, 0x26, 0x9e, 0x09, + 0x38, 0x4e, 0xd5, 0xa0, 0x54, 0xb6, 0x1c, 0xb7, 0xd9, 0x09, 0x48, 0x4c, 0x65, 0xc8, 0xa4, 0xb2, + 0x9a, 0x80, 0xe3, 0x54, 0x0d, 0xba, 0x8e, 0x4e, 0x57, 0x03, 0x9f, 0x1e, 0x5e, 0x32, 0x98, 0x8b, + 0xb2, 0xdb, 0x1d, 0x96, 0x7e, 0xa7, 0x5d, 0xc2, 0x9e, 0x09, 0xcb, 0x46, 0x4e, 0xc1, 0x50, 0xda, + 0xd7, 0x84, 0xc7, 0xa9, 0xa4, 0x82, 0x9e, 0x83, 0x51, 0x91, 0xd5, 0x80, 0x99, 0x97, 0xf2, 0xa9, + 0x63, 0x46, 0x06, 0xe5, 0xb8, 0x18, 0xeb, 0x38, 0xf6, 0xf7, 0x15, 0x60, 0x26, 0xc3, 0x3f, 0x80, + 0x1f, 0x55, 0xdb, 0x6e, 0x18, 0xa9, 0xfc, 0x78, 0xda, 0x51, 0xc5, 0xcb, 0xb1, 0xc2, 0xa0, 0xfb, + 0x81, 0x1f, 0x86, 0xc9, 0x03, 0x50, 0xd8, 0xdf, 0x0a, 0xe8, 0x11, 0x33, 0xcd, 0x5d, 0x84, 0x81, + 0x4e, 0x48, 0x64, 0xc0, 0x42, 0x75, 0x7e, 0x33, 0xdd, 0x13, 0x83, 0x50, 0xf6, 0x78, 0x5b, 0xa9, + 0x71, 0x34, 0xf6, 0x98, 0x2b, 0x72, 0x38, 0x8c, 0x76, 0x2e, 0x22, 0x9e, 0xe3, 0x45, 0x82, 0x89, + 0x8e, 0x23, 0x6f, 0xb1, 0x52, 0x2c, 0xa0, 0xf6, 0x97, 0x8a, 0x70, 0x36, 0xd7, 0x63, 0x88, 0x76, + 0xbd, 0xe5, 0x7b, 0x6e, 0xe4, 0x2b, 0xeb, 0x0d, 0x1e, 0x6d, 0x8b, 0xb4, 0x77, 0xd6, 0x44, 0x39, + 0x56, 0x18, 0xe8, 0x32, 0x0c, 0x32, 0xc9, 0x55, 0x2a, 0x53, 0xe0, 0x52, 0x99, 0x87, 0x5f, 0xe1, + 0xe0, 0xbe, 0xb3, 0xb0, 0x5e, 0x82, 0x81, 0xb6, 0xef, 0x37, 0x93, 0x87, 0x16, 0xed, 0xae, 0xef, + 0x37, 0x31, 0x03, 0xa2, 0x0f, 0x88, 0xf1, 0x4a, 0x98, 0x2b, 0x60, 0xa7, 0xe1, 0x87, 0xda, 0xa0, + 0x3d, 0x05, 0xc3, 0xbb, 0x64, 0x3f, 0x70, 0xbd, 0xed, 0xa4, 0x19, 0xcb, 0x0d, 0x5e, 0x8c, 0x25, + 0xdc, 0x4c, 0xfa, 0x34, 0x7c, 0xdc, 0xe9, 0x53, 0x47, 0x7a, 0x5e, 0x81, 0x3f, 0x50, 0x84, 0x49, + 0xbc, 0x54, 0x7e, 0x6f, 0x22, 0x6e, 0xa7, 0x27, 0xe2, 0xb8, 0xd3, 0xa7, 0xf6, 0x9e, 0x8d, 0x5f, + 0xb0, 0x60, 0x92, 0xe5, 0x56, 0x10, 0x71, 0x9a, 0x5c, 0xdf, 0x3b, 0x01, 0x16, 0xef, 0x12, 0x0c, + 0x06, 0xb4, 0xd1, 0x64, 0x8a, 0x40, 0xd6, 0x13, 0xcc, 0x61, 0xe8, 0x1c, 0x0c, 0xb0, 0x2e, 0xd0, + 0xc9, 0x1b, 0xe3, 0xd9, 0x95, 0xca, 0x4e, 0xe4, 0x60, 0x56, 0xca, 0x82, 0x8f, 0x60, 0xd2, 0x6e, + 0xba, 0xbc, 0xd3, 0xb1, 0x5e, 0xf1, 0xdd, 0xe1, 0x4b, 0x9c, 0xd9, 0xb5, 0x77, 0x16, 0x7c, 0x24, + 0x9b, 0x64, 0xf7, 0xe7, 0xd3, 0x1f, 0x17, 0xe0, 0x42, 0x66, 0xbd, 0xbe, 0x83, 0x8f, 0x74, 0xaf, + 0xfd, 0x28, 0x63, 0xf0, 0x17, 0x4f, 0xd0, 0x48, 0x70, 0xa0, 0x5f, 0x0e, 0x73, 0xb0, 0x8f, 0x98, + 0x20, 0x99, 0x43, 0xf6, 0x2e, 0x89, 0x09, 0x92, 0xd9, 0xb7, 0x9c, 0xe7, 0xdf, 0x5f, 0x16, 0x72, + 0xbe, 0x85, 0x3d, 0x04, 0xaf, 0xd0, 0x73, 0x86, 0x01, 0x43, 0xc1, 0x31, 0x8f, 0xf1, 0x33, 0x86, + 0x97, 0x61, 0x05, 0x45, 0x8b, 0x30, 0xd9, 0x72, 0x3d, 0x7a, 0xf8, 0xec, 0x9b, 0x8c, 0x9f, 0x0a, + 0xd9, 0xb4, 0x66, 0x82, 0x71, 0x12, 0x1f, 0xb9, 0x5a, 0xbc, 0x90, 0x42, 0x7e, 0xd2, 0xed, 0xdc, + 0xde, 0x2e, 0x98, 0x3a, 0x57, 0x35, 0x8a, 0x19, 0xb1, 0x43, 0xd6, 0xb4, 0xf7, 0x7f, 0xb1, 0xff, + 0xf7, 0xff, 0x58, 0xf6, 0xdb, 0x7f, 0xee, 0x55, 0x18, 0x7f, 0x68, 0x81, 0xaf, 0xfd, 0xd5, 0x22, + 0x3c, 0xde, 0x65, 0xdb, 0xf3, 0xb3, 0xde, 0x98, 0x03, 0xed, 0xac, 0x4f, 0xcd, 0x43, 0x15, 0x4e, + 0x6d, 0x75, 0x9a, 0xcd, 0x7d, 0x66, 0x87, 0x4f, 0x1a, 0x12, 0x43, 0xf0, 0x94, 0xe7, 0x64, 0x3e, + 0xab, 0xd5, 0x0c, 0x1c, 0x9c, 0x59, 0x93, 0x32, 0xf4, 0xf4, 0x26, 0xd9, 0x57, 0xa4, 0x12, 0x0c, + 0x3d, 0xd6, 0x81, 0xd8, 0xc4, 0x45, 0xd7, 0x60, 0xda, 0xd9, 0x73, 0x5c, 0x1e, 0x74, 0x55, 0x12, + 0xe0, 0x1c, 0xbd, 0x92, 0xd3, 0x2d, 0x26, 0x11, 0x70, 0xba, 0x0e, 0x7a, 0x1d, 0x90, 0x2f, 0x72, + 0xff, 0x5f, 0x23, 0x9e, 0x50, 0x8d, 0xb1, 0xb9, 0x2b, 0xc6, 0x47, 0xc2, 0xad, 0x14, 0x06, 0xce, + 0xa8, 0x95, 0x88, 0xbf, 0x31, 0x94, 0x1f, 0x7f, 0xa3, 0xfb, 0xb9, 0xd8, 0x33, 0xfd, 0xc3, 0x7f, + 0xb4, 0xe8, 0xf5, 0xc5, 0x99, 0x7c, 0x33, 0x8c, 0xdc, 0xab, 0xcc, 0xb4, 0x8d, 0xcb, 0xf0, 0xb4, + 0xa8, 0x11, 0xa7, 0x35, 0xd3, 0xb6, 0x18, 0x88, 0x4d, 0x5c, 0xbe, 0x20, 0xc2, 0xd8, 0x59, 0xd1, + 0x60, 0xf1, 0x45, 0xac, 0x1b, 0x85, 0x81, 0x3e, 0x09, 0xc3, 0x0d, 0x77, 0xcf, 0x0d, 0xfd, 0x40, + 0xac, 0xf4, 0x23, 0xaa, 0x0b, 0xe2, 0x73, 0xb0, 0xcc, 0xc9, 0x60, 0x49, 0xcf, 0xfe, 0x81, 0x02, + 0x8c, 0xcb, 0x16, 0xdf, 0xe8, 0xf8, 0x91, 0x73, 0x02, 0xd7, 0xf2, 0x35, 0xe3, 0x5a, 0xfe, 0x40, + 0xb7, 0x80, 0x3f, 0xac, 0x4b, 0xb9, 0xd7, 0xf1, 0xad, 0xc4, 0x75, 0xfc, 0x64, 0x6f, 0x52, 0xdd, + 0xaf, 0xe1, 0x7f, 0x64, 0xc1, 0xb4, 0x81, 0x7f, 0x02, 0xb7, 0xc1, 0xaa, 0x79, 0x1b, 0x3c, 0xd1, + 0xf3, 0x1b, 0x72, 0x6e, 0x81, 0xef, 0x2e, 0x26, 0xfa, 0xce, 0x4e, 0xff, 0xb7, 0x60, 0x60, 0xc7, + 0x09, 0x1a, 0xdd, 0x02, 0x9c, 0xa7, 0x2a, 0x2d, 0x5c, 0x77, 0x02, 0xa1, 0x1b, 0x7c, 0x46, 0xe5, + 0xbc, 0x76, 0x82, 0xde, 0x7a, 0x41, 0xd6, 0x14, 0x7a, 0x19, 0x86, 0xc2, 0xba, 0xdf, 0x56, 0x96, + 0xf3, 0x17, 0x79, 0x3e, 0x6c, 0x5a, 0x72, 0x78, 0x30, 0x8f, 0xcc, 0xe6, 0x68, 0x31, 0x16, 0xf8, + 0xe8, 0x53, 0x30, 0xce, 0x7e, 0x29, 0x43, 0x9d, 0x62, 0x7e, 0x32, 0xa4, 0x9a, 0x8e, 0xc8, 0xad, + 0xd8, 0x8c, 0x22, 0x6c, 0x92, 0x9a, 0xdb, 0x86, 0x92, 0xfa, 0xac, 0x47, 0xaa, 0x8f, 0xfb, 0xb7, + 0x45, 0x98, 0xc9, 0x58, 0x73, 0x28, 0x34, 0x66, 0xe2, 0xb9, 0x3e, 0x97, 0xea, 0x3b, 0x9c, 0x8b, + 0x90, 0xbd, 0x86, 0x1a, 0x62, 0x6d, 0xf5, 0xdd, 0xe8, 0xed, 0x90, 0x24, 0x1b, 0xa5, 0x45, 0xbd, + 0x1b, 0xa5, 0x8d, 0x9d, 0xd8, 0x50, 0xd3, 0x86, 0x54, 0x4f, 0x1f, 0xe9, 0x9c, 0xfe, 0x59, 0x11, + 0x4e, 0x65, 0xc5, 0x20, 0x43, 0x9f, 0x4f, 0x24, 0xc6, 0x7b, 0xb1, 0xdf, 0xe8, 0x65, 0x3c, 0x5b, + 0x1e, 0x97, 0x01, 0x2f, 0x2d, 0x98, 0xa9, 0xf2, 0x7a, 0x0e, 0xb3, 0x68, 0x93, 0x39, 0xe2, 0x07, + 0x3c, 0xa1, 0xa1, 0x3c, 0x3e, 0x3e, 0xdc, 0x77, 0x07, 0x44, 0x26, 0xc4, 0x30, 0x61, 0x04, 0x20, + 0x8b, 0x7b, 0x1b, 0x01, 0xc8, 0x96, 0xe7, 0x5c, 0x18, 0xd5, 0xbe, 0xe6, 0x91, 0xce, 0xf8, 0x2e, + 0xbd, 0xad, 0xb4, 0x7e, 0x3f, 0xd2, 0x59, 0xff, 0x51, 0x0b, 0x12, 0x76, 0xe1, 0x4a, 0x2c, 0x66, + 0xe5, 0x8a, 0xc5, 0x2e, 0xc2, 0x40, 0xe0, 0x37, 0x49, 0x32, 0x0f, 0x1d, 0xf6, 0x9b, 0x04, 0x33, + 0x08, 0xc5, 0x88, 0x62, 0x61, 0xc7, 0x98, 0xfe, 0x90, 0x13, 0x4f, 0xb4, 0x4b, 0x30, 0xd8, 0x24, + 0x7b, 0xa4, 0x99, 0x4c, 0x17, 0x72, 0x93, 0x16, 0x62, 0x0e, 0xb3, 0x7f, 0x61, 0x00, 0xce, 0x77, + 0x0d, 0x65, 0x41, 0x9f, 0x43, 0xdb, 0x4e, 0x44, 0xee, 0x39, 0xfb, 0xc9, 0xb8, 0xfe, 0xd7, 0x78, + 0x31, 0x96, 0x70, 0xe6, 0xb9, 0xc3, 0xc3, 0xf3, 0x26, 0x84, 0x88, 0x22, 0x2a, 0xaf, 0x80, 0x9a, + 0x42, 0xa9, 0xe2, 0x71, 0x08, 0xa5, 0x9e, 0x07, 0x08, 0xc3, 0x26, 0xb7, 0x9e, 0x69, 0x08, 0x97, + 0xa0, 0x38, 0x8c, 0x73, 0xed, 0xa6, 0x80, 0x60, 0x0d, 0x0b, 0x95, 0x61, 0xaa, 0x1d, 0xf8, 0x11, + 0x97, 0xc9, 0x96, 0xb9, 0x81, 0xd9, 0xa0, 0x19, 0x45, 0xa0, 0x9a, 0x80, 0xe3, 0x54, 0x0d, 0xf4, + 0x12, 0x8c, 0x8a, 0xc8, 0x02, 0x55, 0xdf, 0x6f, 0x0a, 0x31, 0x90, 0xb2, 0xb9, 0xaa, 0xc5, 0x20, + 0xac, 0xe3, 0x69, 0xd5, 0x98, 0xa0, 0x77, 0x38, 0xb3, 0x1a, 0x17, 0xf6, 0x6a, 0x78, 0x89, 0x78, + 0x84, 0x23, 0x7d, 0xc5, 0x23, 0x8c, 0x05, 0x63, 0xa5, 0xbe, 0x75, 0x5b, 0xd0, 0x53, 0x94, 0xf4, + 0xb3, 0x03, 0x30, 0x23, 0x16, 0xce, 0xa3, 0x5e, 0x2e, 0xb7, 0xd3, 0xcb, 0xe5, 0x38, 0x44, 0x67, + 0xef, 0xad, 0x99, 0x93, 0x5e, 0x33, 0x3f, 0x68, 0x81, 0xc9, 0x5e, 0xa1, 0xff, 0x3b, 0x37, 0x31, + 0xca, 0x4b, 0xb9, 0xec, 0x5a, 0x43, 0x5e, 0x20, 0xef, 0x30, 0x45, 0x8a, 0xfd, 0xef, 0x2d, 0x78, + 0xa2, 0x27, 0x45, 0xb4, 0x02, 0x25, 0xc6, 0x03, 0x6a, 0xaf, 0xb3, 0x27, 0x95, 0x01, 0xaa, 0x04, + 0xe4, 0xb0, 0xa4, 0x71, 0x4d, 0xb4, 0x92, 0xca, 0x40, 0xf3, 0x54, 0x46, 0x06, 0x9a, 0xd3, 0xc6, + 0xf0, 0x3c, 0x64, 0x0a, 0x9a, 0xef, 0xa7, 0x37, 0x8e, 0xe1, 0xfc, 0x81, 0x3e, 0x6c, 0x88, 0xfd, + 0xec, 0x84, 0xd8, 0x0f, 0x99, 0xd8, 0xda, 0x1d, 0xf2, 0x71, 0x98, 0x62, 0x21, 0x87, 0x98, 0x39, + 0xb4, 0x70, 0x4b, 0x29, 0xc4, 0x26, 0x8f, 0x37, 0x13, 0x30, 0x9c, 0xc2, 0xb6, 0xff, 0xa8, 0x08, + 0x43, 0x7c, 0xfb, 0x9d, 0xc0, 0x9b, 0xf0, 0x69, 0x28, 0xb9, 0xad, 0x56, 0x87, 0x27, 0x15, 0x19, + 0xe4, 0xbe, 0xa8, 0x74, 0x9e, 0x2a, 0xb2, 0x10, 0xc7, 0x70, 0xb4, 0x2a, 0x24, 0xce, 0x5d, 0xa2, + 0x1a, 0xf2, 0x8e, 0x2f, 0x94, 0x9d, 0xc8, 0xe1, 0x0c, 0x8e, 0xba, 0x67, 0x63, 0xd9, 0x34, 0xfa, + 0x0c, 0x40, 0x18, 0x05, 0xae, 0xb7, 0x4d, 0xcb, 0x44, 0x10, 0xcf, 0x0f, 0x76, 0xa1, 0x56, 0x53, + 0xc8, 0x9c, 0x66, 0x7c, 0xe6, 0x28, 0x00, 0xd6, 0x28, 0xa2, 0x05, 0xe3, 0xa6, 0x9f, 0x4b, 0xcc, + 0x1d, 0x70, 0xaa, 0xf1, 0x9c, 0xcd, 0x7d, 0x04, 0x4a, 0x8a, 0x78, 0x2f, 0xf9, 0xd3, 0x98, 0xce, + 0x16, 0x7d, 0x0c, 0x26, 0x13, 0x7d, 0x3b, 0x92, 0xf8, 0xea, 0x17, 0x2d, 0x98, 0xe4, 0x9d, 0x59, + 0xf1, 0xf6, 0xc4, 0x6d, 0xf0, 0x36, 0x9c, 0x6a, 0x66, 0x9c, 0xca, 0x62, 0xfa, 0xfb, 0x3f, 0xc5, + 0x95, 0xb8, 0x2a, 0x0b, 0x8a, 0x33, 0xdb, 0x40, 0x57, 0xe8, 0x8e, 0xa3, 0xa7, 0xae, 0xd3, 0x14, + 0xae, 0xa9, 0x63, 0x7c, 0xb7, 0xf1, 0x32, 0xac, 0xa0, 0xf6, 0xef, 0x5a, 0x30, 0xcd, 0x7b, 0x7e, + 0x83, 0xec, 0xab, 0xb3, 0xe9, 0xeb, 0xd9, 0x77, 0x91, 0xce, 0xaa, 0x90, 0x93, 0xce, 0x4a, 0xff, + 0xb4, 0x62, 0xd7, 0x4f, 0xfb, 0x19, 0x0b, 0xc4, 0x0a, 0x39, 0x01, 0x21, 0xc4, 0xb7, 0x9a, 0x42, + 0x88, 0xb9, 0xfc, 0x4d, 0x90, 0x23, 0x7d, 0xf8, 0x0b, 0x0b, 0xa6, 0x38, 0x42, 0xac, 0x2d, 0xff, + 0xba, 0xce, 0x43, 0x3f, 0x49, 0x6f, 0x6f, 0x90, 0xfd, 0x0d, 0xbf, 0xea, 0x44, 0x3b, 0xd9, 0x1f, + 0x65, 0x4c, 0xd6, 0x40, 0xd7, 0xc9, 0x6a, 0xc8, 0x0d, 0x74, 0x84, 0x4c, 0xda, 0x47, 0xce, 0xf6, + 0x60, 0x7f, 0xcd, 0x02, 0xc4, 0x9b, 0x31, 0x18, 0x37, 0xca, 0x0e, 0xb1, 0x52, 0xed, 0xa2, 0x8b, + 0x8f, 0x26, 0x05, 0xc1, 0x1a, 0xd6, 0xb1, 0x0c, 0x4f, 0xc2, 0xe4, 0xa1, 0xd8, 0xdb, 0xe4, 0xe1, + 0x08, 0x23, 0xfa, 0x2f, 0x87, 0x20, 0xe9, 0x00, 0x83, 0xee, 0xc0, 0x58, 0xdd, 0x69, 0x3b, 0x9b, + 0x6e, 0xd3, 0x8d, 0x5c, 0x12, 0x76, 0xb3, 0x95, 0x5a, 0xd6, 0xf0, 0x84, 0x92, 0x5a, 0x2b, 0xc1, + 0x06, 0x1d, 0xb4, 0x00, 0xd0, 0x0e, 0xdc, 0x3d, 0xb7, 0x49, 0xb6, 0x99, 0xac, 0x84, 0x39, 0xc3, + 0x73, 0x03, 0x20, 0x59, 0x8a, 0x35, 0x8c, 0x0c, 0x6f, 0xe3, 0xe2, 0x23, 0xf6, 0x36, 0x86, 0x13, + 0xf3, 0x36, 0x1e, 0x38, 0x92, 0xb7, 0xf1, 0xc8, 0x91, 0xbd, 0x8d, 0x07, 0xfb, 0xf2, 0x36, 0xc6, + 0x70, 0x46, 0xf2, 0x9e, 0xf4, 0xff, 0xaa, 0xdb, 0x24, 0xe2, 0xc1, 0xc1, 0x3d, 0xf8, 0xe7, 0x1e, + 0x1c, 0xcc, 0x9f, 0xc1, 0x99, 0x18, 0x38, 0xa7, 0x26, 0xfa, 0x04, 0xcc, 0x3a, 0xcd, 0xa6, 0x7f, + 0x4f, 0x4d, 0xea, 0x4a, 0x58, 0x77, 0x9a, 0x5c, 0x09, 0x31, 0xcc, 0xa8, 0x9e, 0x7b, 0x70, 0x30, + 0x3f, 0xbb, 0x98, 0x83, 0x83, 0x73, 0x6b, 0xa3, 0x8f, 0x42, 0xa9, 0x1d, 0xf8, 0xf5, 0x35, 0xcd, + 0x4b, 0xef, 0x02, 0x1d, 0xc0, 0xaa, 0x2c, 0x3c, 0x3c, 0x98, 0x1f, 0x57, 0x7f, 0xd8, 0x85, 0x1f, + 0x57, 0xc8, 0x70, 0x1f, 0x1e, 0x3d, 0x56, 0xf7, 0xe1, 0x5d, 0x98, 0xa9, 0x91, 0xc0, 0x65, 0x79, + 0xb7, 0x1b, 0xf1, 0xf9, 0xb4, 0x01, 0xa5, 0x20, 0x71, 0x22, 0xf7, 0x15, 0x69, 0x50, 0x0b, 0xbb, + 0x2f, 0x4f, 0xe0, 0x98, 0x90, 0xfd, 0x3f, 0x2d, 0x18, 0x16, 0x0e, 0x2f, 0x27, 0xc0, 0x35, 0x2e, + 0x1a, 0x9a, 0x84, 0xf9, 0xec, 0x01, 0x63, 0x9d, 0xc9, 0xd5, 0x21, 0x54, 0x12, 0x3a, 0x84, 0x27, + 0xba, 0x11, 0xe9, 0xae, 0x3d, 0xf8, 0x6b, 0x45, 0xca, 0xbd, 0x1b, 0xae, 0x97, 0x8f, 0x7e, 0x08, + 0xd6, 0x61, 0x38, 0x14, 0xae, 0x7f, 0x85, 0x7c, 0x5b, 0xf5, 0xe4, 0x24, 0xc6, 0x76, 0x6c, 0xc2, + 0xd9, 0x4f, 0x12, 0xc9, 0xf4, 0x29, 0x2c, 0x3e, 0x42, 0x9f, 0xc2, 0x5e, 0xce, 0xa9, 0x03, 0xc7, + 0xe1, 0x9c, 0x6a, 0x7f, 0x85, 0xdd, 0x9c, 0x7a, 0xf9, 0x09, 0x30, 0x55, 0xd7, 0xcc, 0x3b, 0xd6, + 0xee, 0xb2, 0xb2, 0x44, 0xa7, 0x72, 0x98, 0xab, 0x9f, 0xb7, 0xe0, 0x7c, 0xc6, 0x57, 0x69, 0x9c, + 0xd6, 0x33, 0x30, 0xe2, 0x74, 0x1a, 0xae, 0xda, 0xcb, 0x9a, 0x3e, 0x71, 0x51, 0x94, 0x63, 0x85, + 0x81, 0x96, 0x61, 0x9a, 0xdc, 0x6f, 0xbb, 0x5c, 0x95, 0xaa, 0x1b, 0x9b, 0x16, 0xb9, 0x97, 0xd4, + 0x4a, 0x12, 0x88, 0xd3, 0xf8, 0x2a, 0x20, 0x48, 0x31, 0x37, 0x20, 0xc8, 0xdf, 0xb5, 0x60, 0x54, + 0x39, 0xbf, 0x3d, 0xf2, 0xd1, 0xfe, 0xb8, 0x39, 0xda, 0x8f, 0x77, 0x19, 0xed, 0x9c, 0x61, 0xfe, + 0x9d, 0x82, 0xea, 0x6f, 0xd5, 0x0f, 0xa2, 0x3e, 0x38, 0xb8, 0x97, 0x61, 0xa4, 0x1d, 0xf8, 0x91, + 0x5f, 0xf7, 0x9b, 0x82, 0x81, 0x3b, 0x17, 0x47, 0xc6, 0xe1, 0xe5, 0x87, 0xda, 0x6f, 0xac, 0xb0, + 0x29, 0xef, 0xe4, 0xb4, 0xdb, 0x12, 0x20, 0x6d, 0xd0, 0x58, 0xdc, 0xd8, 0xb8, 0x18, 0xeb, 0x38, + 0x6c, 0xc0, 0xfd, 0x20, 0x12, 0x7c, 0x56, 0x3c, 0xe0, 0x7e, 0x10, 0x61, 0x06, 0x41, 0x0d, 0x80, + 0xc8, 0x09, 0xb6, 0x49, 0x44, 0xcb, 0x44, 0xf0, 0xae, 0xfc, 0xf3, 0xa6, 0x13, 0xb9, 0xcd, 0x05, + 0xd7, 0x8b, 0xc2, 0x28, 0x58, 0xa8, 0x78, 0xd1, 0xad, 0x80, 0x3f, 0x21, 0xb5, 0xe8, 0x38, 0x8a, + 0x16, 0xd6, 0xe8, 0x4a, 0x47, 0x6f, 0xd6, 0xc6, 0xa0, 0x69, 0xcc, 0xb0, 0x2e, 0xca, 0xb1, 0xc2, + 0xb0, 0x3f, 0xc2, 0x6e, 0x1f, 0x36, 0xa6, 0x47, 0x0b, 0x27, 0xf3, 0xeb, 0xa0, 0x66, 0x83, 0x69, + 0x32, 0xcb, 0x7a, 0xd0, 0x9a, 0xee, 0x87, 0x3d, 0x6d, 0x58, 0xf7, 0xd7, 0x8a, 0x23, 0xdb, 0xa0, + 0x6f, 0x4b, 0x19, 0xa8, 0x3c, 0xdb, 0xe3, 0xd6, 0x38, 0x82, 0x49, 0x0a, 0x4b, 0x22, 0xc1, 0x42, + 0xec, 0x57, 0xaa, 0x62, 0x5f, 0x68, 0x49, 0x24, 0x04, 0x00, 0xc7, 0x38, 0x94, 0x99, 0x52, 0x7f, + 0xc2, 0x59, 0x14, 0x07, 0x53, 0x54, 0xd8, 0x21, 0xd6, 0x30, 0xd0, 0x55, 0x21, 0x50, 0xe0, 0x7a, + 0x81, 0xc7, 0x13, 0x02, 0x05, 0x39, 0x5c, 0x9a, 0x14, 0xe8, 0x39, 0x18, 0x55, 0x79, 0x64, 0xab, + 0x3c, 0x3d, 0xa9, 0x58, 0x66, 0x2b, 0x71, 0x31, 0xd6, 0x71, 0xd0, 0x06, 0x4c, 0x86, 0x5c, 0xce, + 0xa6, 0x22, 0xdc, 0x72, 0x79, 0xe5, 0x07, 0xa5, 0x15, 0x50, 0xcd, 0x04, 0x1f, 0xb2, 0x22, 0x7e, + 0x3a, 0x49, 0x67, 0xec, 0x24, 0x09, 0xf4, 0x1a, 0x4c, 0x34, 0x7d, 0xa7, 0xb1, 0xe4, 0x34, 0x1d, + 0xaf, 0xce, 0xc6, 0x67, 0xc4, 0x4c, 0x47, 0x78, 0xd3, 0x80, 0xe2, 0x04, 0x36, 0x65, 0xde, 0xf4, + 0x12, 0x11, 0x95, 0xd9, 0xf1, 0xb6, 0x49, 0x28, 0xb2, 0x82, 0x32, 0xe6, 0xed, 0x66, 0x0e, 0x0e, + 0xce, 0xad, 0x8d, 0x5e, 0x86, 0x31, 0xf9, 0xf9, 0x5a, 0xec, 0x82, 0xd8, 0xf1, 0x41, 0x83, 0x61, + 0x03, 0x13, 0xdd, 0x83, 0xd3, 0xf2, 0xff, 0x46, 0xe0, 0x6c, 0x6d, 0xb9, 0x75, 0xe1, 0xd0, 0xcb, + 0xbd, 0x12, 0x17, 0xa5, 0xeb, 0xdc, 0x4a, 0x16, 0xd2, 0xe1, 0xc1, 0xfc, 0x45, 0x31, 0x6a, 0x99, + 0x70, 0x36, 0x89, 0xd9, 0xf4, 0xd1, 0x1a, 0xcc, 0xec, 0x10, 0xa7, 0x19, 0xed, 0x2c, 0xef, 0x90, + 0xfa, 0xae, 0xdc, 0x74, 0x2c, 0x22, 0x82, 0xe6, 0x2e, 0x70, 0x3d, 0x8d, 0x82, 0xb3, 0xea, 0xa1, + 0x37, 0x61, 0xb6, 0xdd, 0xd9, 0x6c, 0xba, 0xe1, 0xce, 0xba, 0x1f, 0x31, 0x53, 0x20, 0x95, 0x96, + 0x56, 0x84, 0x4e, 0x50, 0x31, 0x27, 0xaa, 0x39, 0x78, 0x38, 0x97, 0x02, 0x7a, 0x1b, 0x4e, 0x27, + 0x16, 0x83, 0x70, 0x1e, 0x9f, 0xc8, 0x8f, 0x71, 0x5f, 0xcb, 0xaa, 0x20, 0xe2, 0x30, 0x64, 0x81, + 0x70, 0x76, 0x13, 0xe8, 0x15, 0x00, 0xb7, 0xbd, 0xea, 0xb4, 0xdc, 0x26, 0x7d, 0x2e, 0xce, 0xb0, + 0x75, 0x42, 0x9f, 0x0e, 0x50, 0xa9, 0xca, 0x52, 0x7a, 0x3e, 0x8b, 0x7f, 0xfb, 0x58, 0xc3, 0xa6, + 0x0f, 0x17, 0x2d, 0x20, 0x69, 0x38, 0x3b, 0x15, 0xdb, 0x3b, 0x6b, 0x51, 0x4b, 0x43, 0x6c, 0x60, + 0xa1, 0x2a, 0x4c, 0x08, 0x1a, 0xfb, 0x62, 0x31, 0x4c, 0x2b, 0xef, 0xee, 0x09, 0xd9, 0x8e, 0x5a, + 0x01, 0xc8, 0x2c, 0x61, 0x73, 0x9e, 0xa8, 0xff, 0xce, 0x0c, 0xd3, 0xde, 0xa2, 0x95, 0x35, 0xde, + 0x15, 0x7d, 0x16, 0xc6, 0xf4, 0x9d, 0x20, 0xee, 0xe1, 0xcb, 0xd9, 0xac, 0x9d, 0xb6, 0x63, 0x38, + 0xe7, 0xab, 0x76, 0x85, 0x0e, 0xc3, 0x06, 0x45, 0x9b, 0x40, 0xf6, 0x1c, 0xa1, 0x9b, 0x30, 0x52, + 0x6f, 0xba, 0xc4, 0x8b, 0x2a, 0xd5, 0x6e, 0xd1, 0xb9, 0x96, 0x05, 0x8e, 0x98, 0x74, 0x11, 0xd8, + 0x9c, 0x97, 0x61, 0x45, 0xc1, 0xfe, 0xb5, 0x02, 0xcc, 0xf7, 0x88, 0x92, 0x9f, 0xd0, 0x9f, 0x58, + 0x7d, 0xe9, 0x4f, 0x16, 0x65, 0xa2, 0xe0, 0xf5, 0x84, 0x68, 0x26, 0x91, 0x04, 0x38, 0x16, 0xd0, + 0x24, 0xf1, 0xfb, 0xb6, 0x67, 0xd7, 0x55, 0x30, 0x03, 0x3d, 0x3d, 0x32, 0x0c, 0xd5, 0xeb, 0x60, + 0xff, 0xef, 0xb5, 0x5c, 0x35, 0x9a, 0xfd, 0x95, 0x02, 0x9c, 0x56, 0x43, 0xf8, 0xcd, 0x3b, 0x70, + 0xb7, 0xd3, 0x03, 0x77, 0x0c, 0x4a, 0x48, 0xfb, 0x16, 0x0c, 0xf1, 0x70, 0x63, 0x7d, 0xf0, 0x89, + 0x97, 0xcc, 0xc8, 0x9c, 0x8a, 0x35, 0x31, 0xa2, 0x73, 0x7e, 0xaf, 0x05, 0x93, 0x1b, 0xcb, 0xd5, + 0x9a, 0x5f, 0xdf, 0x25, 0xd1, 0x22, 0xe7, 0xeb, 0xb1, 0xe0, 0xf9, 0xac, 0x87, 0xe4, 0xe5, 0xb2, + 0xb8, 0xc4, 0x8b, 0x30, 0xb0, 0xe3, 0x87, 0x51, 0xd2, 0x42, 0xe1, 0xba, 0x1f, 0x46, 0x98, 0x41, + 0xec, 0xdf, 0xb3, 0x60, 0x90, 0xa5, 0xc6, 0x97, 0xd2, 0x6c, 0x2b, 0x47, 0x9a, 0xdd, 0xcf, 0x77, + 0xa1, 0x97, 0x60, 0x88, 0x6c, 0x6d, 0x91, 0x7a, 0x24, 0x66, 0x55, 0xba, 0x94, 0x0f, 0xad, 0xb0, + 0x52, 0xca, 0xb8, 0xb0, 0xc6, 0xf8, 0x5f, 0x2c, 0x90, 0xd1, 0x5d, 0x28, 0x45, 0x6e, 0x8b, 0x2c, + 0x36, 0x1a, 0x42, 0xc7, 0xfb, 0x10, 0x6e, 0xf1, 0x1b, 0x92, 0x00, 0x8e, 0x69, 0xd9, 0x5f, 0x2a, + 0x00, 0xc4, 0x71, 0x5a, 0x7a, 0x7d, 0xe2, 0x52, 0x4a, 0xfb, 0x77, 0x39, 0x43, 0xfb, 0x87, 0x62, + 0x82, 0x19, 0xaa, 0x3f, 0x35, 0x4c, 0xc5, 0xbe, 0x86, 0x69, 0xe0, 0x28, 0xc3, 0xb4, 0x0c, 0xd3, + 0x71, 0x9c, 0x19, 0x33, 0xcc, 0x16, 0x7b, 0xcb, 0x6d, 0x24, 0x81, 0x38, 0x8d, 0x6f, 0x13, 0xb8, + 0xa8, 0xc2, 0x6d, 0x88, 0xbb, 0x86, 0x99, 0x10, 0xeb, 0xda, 0xd4, 0x1e, 0xe3, 0x14, 0xab, 0x37, + 0x0b, 0xb9, 0xea, 0xcd, 0x9f, 0xb0, 0xe0, 0x54, 0xb2, 0x1d, 0xe6, 0xd3, 0xf9, 0x45, 0x0b, 0x4e, + 0x33, 0x25, 0x2f, 0x6b, 0x35, 0xad, 0x52, 0x7e, 0xb1, 0x6b, 0x08, 0x91, 0x9c, 0x1e, 0xc7, 0xb1, + 0x0b, 0xd6, 0xb2, 0x48, 0xe3, 0xec, 0x16, 0xed, 0x7f, 0x57, 0x80, 0xd9, 0xbc, 0xd8, 0x23, 0xcc, + 0xc3, 0xc0, 0xb9, 0x5f, 0xdb, 0x25, 0xf7, 0x84, 0x1d, 0x77, 0xec, 0x61, 0xc0, 0x8b, 0xb1, 0x84, + 0x27, 0x03, 0x9f, 0x17, 0xfa, 0x0b, 0x7c, 0x8e, 0x76, 0x60, 0xfa, 0xde, 0x0e, 0xf1, 0x6e, 0x7b, + 0xa1, 0x13, 0xb9, 0xe1, 0x96, 0xcb, 0x14, 0xa2, 0x7c, 0xdd, 0xbc, 0x22, 0xad, 0xad, 0xef, 0x26, + 0x11, 0x0e, 0x0f, 0xe6, 0xcf, 0x1b, 0x05, 0x71, 0x97, 0xf9, 0x41, 0x82, 0xd3, 0x44, 0xd3, 0x71, + 0xe3, 0x07, 0x1e, 0x61, 0xdc, 0x78, 0xfb, 0x8b, 0x16, 0x9c, 0xcd, 0x4d, 0x56, 0x89, 0xae, 0xc0, + 0x88, 0xd3, 0x76, 0xb9, 0x4c, 0x59, 0x1c, 0xa3, 0x4c, 0x76, 0x51, 0xad, 0x70, 0x89, 0xb2, 0x82, + 0xaa, 0x24, 0xda, 0x85, 0xdc, 0x24, 0xda, 0x3d, 0x73, 0x62, 0xdb, 0xdf, 0x63, 0x81, 0xf0, 0x8e, + 0xec, 0xe3, 0xec, 0xfe, 0x14, 0x8c, 0xed, 0xa5, 0x73, 0xcb, 0x5c, 0xcc, 0x77, 0x17, 0x15, 0x19, + 0x65, 0x14, 0xaf, 0x64, 0xe4, 0x91, 0x31, 0x68, 0xd9, 0x0d, 0x10, 0xd0, 0x32, 0x61, 0x12, 0xd3, + 0xde, 0xbd, 0x79, 0x1e, 0xa0, 0xc1, 0x70, 0xb5, 0x4c, 0xe4, 0xea, 0x66, 0x2e, 0x2b, 0x08, 0xd6, + 0xb0, 0xec, 0x7f, 0x5d, 0x80, 0x51, 0x99, 0xcb, 0xa4, 0xe3, 0xf5, 0x23, 0xd7, 0x38, 0x52, 0x72, + 0x43, 0x96, 0xba, 0x9f, 0x12, 0xae, 0xc6, 0xe2, 0xa0, 0x38, 0x75, 0xbf, 0x04, 0xe0, 0x18, 0x87, + 0xee, 0xa2, 0xb0, 0xb3, 0xc9, 0xd0, 0x13, 0xbe, 0x7c, 0x35, 0x5e, 0x8c, 0x25, 0x1c, 0x7d, 0x02, + 0xa6, 0x78, 0xbd, 0xc0, 0x6f, 0x3b, 0xdb, 0x5c, 0x58, 0x3f, 0xa8, 0x9c, 0xf0, 0xa7, 0xd6, 0x12, + 0xb0, 0xc3, 0x83, 0xf9, 0x53, 0xc9, 0x32, 0xa6, 0x85, 0x4a, 0x51, 0x61, 0x36, 0x39, 0xbc, 0x11, + 0xba, 0xfb, 0x53, 0xa6, 0x3c, 0x31, 0x08, 0xeb, 0x78, 0xf6, 0x67, 0x01, 0xa5, 0xb3, 0xba, 0xa0, + 0xd7, 0xb9, 0x21, 0xa6, 0x1b, 0x90, 0x46, 0x37, 0xad, 0x94, 0xee, 0x6a, 0x2e, 0xdd, 0x70, 0x78, + 0x2d, 0xac, 0xea, 0xdb, 0xff, 0x5f, 0x11, 0xa6, 0x92, 0x8e, 0xc7, 0xe8, 0x3a, 0x0c, 0x71, 0xd6, + 0x43, 0x90, 0xef, 0x62, 0xf4, 0xa0, 0xb9, 0x2b, 0xb3, 0x43, 0x58, 0x70, 0x2f, 0xa2, 0x3e, 0x7a, + 0x13, 0x46, 0x1b, 0xfe, 0x3d, 0xef, 0x9e, 0x13, 0x34, 0x16, 0xab, 0x15, 0xb1, 0x9c, 0x33, 0x5f, + 0x61, 0xe5, 0x18, 0x4d, 0x77, 0x81, 0x66, 0x0a, 0xbe, 0x18, 0x84, 0x75, 0x72, 0x68, 0x83, 0x05, + 0xa1, 0xde, 0x72, 0xb7, 0xd7, 0x9c, 0x76, 0x37, 0xab, 0xfc, 0x65, 0x89, 0xa4, 0x51, 0x1e, 0x17, + 0x91, 0xaa, 0x39, 0x00, 0xc7, 0x84, 0xd0, 0xe7, 0x61, 0x26, 0xcc, 0x91, 0x0d, 0xe7, 0x25, 0xf9, + 0xea, 0x26, 0x2e, 0x5d, 0x7a, 0x8c, 0xbe, 0x8f, 0xb3, 0xa4, 0xc8, 0x59, 0xcd, 0xd8, 0x3f, 0x72, + 0x0a, 0x8c, 0x4d, 0x6c, 0xe4, 0x7c, 0xb4, 0x8e, 0x29, 0xe7, 0x23, 0x86, 0x11, 0xd2, 0x6a, 0x47, + 0xfb, 0x65, 0x37, 0xe8, 0x96, 0x09, 0x79, 0x45, 0xe0, 0xa4, 0x69, 0x4a, 0x08, 0x56, 0x74, 0xb2, + 0x13, 0x73, 0x16, 0xbf, 0x8e, 0x89, 0x39, 0x07, 0x4e, 0x30, 0x31, 0xe7, 0x3a, 0x0c, 0x6f, 0xbb, + 0x11, 0x26, 0x6d, 0x5f, 0x30, 0xfd, 0x99, 0xeb, 0xf0, 0x1a, 0x47, 0x49, 0xa7, 0x80, 0x13, 0x00, + 0x2c, 0x89, 0xa0, 0xd7, 0xd5, 0x0e, 0x1c, 0xca, 0x7f, 0x33, 0xa7, 0xb5, 0xf3, 0x99, 0x7b, 0x50, + 0xa4, 0xdf, 0x1c, 0x7e, 0xd8, 0xf4, 0x9b, 0xab, 0x32, 0x69, 0xe6, 0x48, 0xbe, 0x0b, 0x0d, 0xcb, + 0x89, 0xd9, 0x23, 0x55, 0xe6, 0x1d, 0x3d, 0xd1, 0x68, 0x29, 0xff, 0x24, 0x50, 0x39, 0x44, 0xfb, + 0x4c, 0x2f, 0xfa, 0x3d, 0x16, 0x9c, 0x6e, 0x67, 0xe5, 0xdc, 0x15, 0x8a, 0xec, 0x97, 0xfa, 0x4e, + 0xeb, 0x6b, 0x34, 0xc8, 0x04, 0x40, 0x99, 0x68, 0x38, 0xbb, 0x39, 0x3a, 0xd0, 0xc1, 0x66, 0x43, + 0x28, 0x54, 0x2f, 0xe5, 0xe4, 0x29, 0xed, 0x92, 0x9d, 0x74, 0x23, 0x23, 0x27, 0xe6, 0xfb, 0xf3, + 0x72, 0x62, 0xf6, 0x9d, 0x09, 0xf3, 0x75, 0x95, 0xa1, 0x74, 0x3c, 0x7f, 0x29, 0xf1, 0xfc, 0xa3, + 0x3d, 0xf3, 0x92, 0xbe, 0xae, 0xf2, 0x92, 0x76, 0x89, 0x30, 0xca, 0xb3, 0x8e, 0xf6, 0xcc, 0x46, + 0xaa, 0x65, 0x14, 0x9d, 0x3c, 0x9e, 0x8c, 0xa2, 0xc6, 0x55, 0xc3, 0x93, 0x5a, 0x3e, 0xdd, 0xe3, + 0xaa, 0x31, 0xe8, 0x76, 0xbf, 0x6c, 0x78, 0xf6, 0xd4, 0xe9, 0x87, 0xca, 0x9e, 0x7a, 0x47, 0xcf, + 0x46, 0x8a, 0x7a, 0xa4, 0xdb, 0xa4, 0x48, 0x7d, 0xe6, 0x20, 0xbd, 0xa3, 0x5f, 0x80, 0x33, 0xf9, + 0x74, 0xd5, 0x3d, 0x97, 0xa6, 0x9b, 0x79, 0x05, 0xa6, 0x72, 0x9b, 0x9e, 0x3a, 0x99, 0xdc, 0xa6, + 0xa7, 0x8f, 0x3d, 0xb7, 0xe9, 0x99, 0x13, 0xc8, 0x6d, 0xfa, 0xd8, 0x09, 0xe6, 0x36, 0xbd, 0xc3, + 0xac, 0x3f, 0x78, 0x8c, 0x19, 0x11, 0x11, 0x35, 0x3b, 0xfa, 0x66, 0x56, 0x20, 0x1a, 0xfe, 0x71, + 0x0a, 0x84, 0x63, 0x52, 0x19, 0x39, 0x53, 0x67, 0x1f, 0x41, 0xce, 0xd4, 0xf5, 0x38, 0x67, 0xea, + 0xd9, 0xfc, 0xa9, 0xce, 0xf0, 0x17, 0xc8, 0xc9, 0x94, 0x7a, 0x47, 0xcf, 0x70, 0xfa, 0x78, 0x17, + 0x11, 0x7f, 0x96, 0xe0, 0xb1, 0x4b, 0x5e, 0xd3, 0xd7, 0x78, 0x5e, 0xd3, 0x73, 0xf9, 0x27, 0x79, + 0xf2, 0xba, 0x33, 0xb2, 0x99, 0xd2, 0x7e, 0xa9, 0xd8, 0x7b, 0x2c, 0xf6, 0x6b, 0x4e, 0xbf, 0x54, + 0xf0, 0xbe, 0x74, 0xbf, 0x14, 0x08, 0xc7, 0xa4, 0xec, 0xef, 0x2b, 0xc0, 0x85, 0xee, 0xfb, 0x2d, + 0x96, 0xa6, 0x56, 0x63, 0x8d, 0x67, 0x42, 0x9a, 0xca, 0xdf, 0x6c, 0x31, 0x56, 0xdf, 0x61, 0xcd, + 0xae, 0xc1, 0xb4, 0x72, 0x34, 0x68, 0xba, 0xf5, 0xfd, 0xf5, 0xf8, 0xe5, 0xab, 0x9c, 0xb3, 0x6b, + 0x49, 0x04, 0x9c, 0xae, 0x83, 0x16, 0x61, 0xd2, 0x28, 0xac, 0x94, 0xc5, 0xdb, 0x4c, 0x89, 0x6f, + 0x6b, 0x26, 0x18, 0x27, 0xf1, 0xed, 0x2f, 0x5b, 0xf0, 0x58, 0x4e, 0x3a, 0xb2, 0xbe, 0xa3, 0x76, + 0x6d, 0xc1, 0x64, 0xdb, 0xac, 0xda, 0x23, 0xb8, 0x9f, 0x91, 0xf4, 0x4c, 0xf5, 0x35, 0x01, 0xc0, + 0x49, 0xa2, 0xf6, 0x9f, 0x5b, 0x70, 0xbe, 0xab, 0xe5, 0x1c, 0xc2, 0x70, 0x66, 0xbb, 0x15, 0x3a, + 0xcb, 0x01, 0x69, 0x10, 0x2f, 0x72, 0x9d, 0x66, 0xad, 0x4d, 0xea, 0x9a, 0x3c, 0x9c, 0x99, 0xa0, + 0x5d, 0x5b, 0xab, 0x2d, 0xa6, 0x31, 0x70, 0x4e, 0x4d, 0xb4, 0x0a, 0x28, 0x0d, 0x11, 0x33, 0xcc, + 0xa2, 0x08, 0xa7, 0xe9, 0xe1, 0x8c, 0x1a, 0xe8, 0x23, 0x30, 0xae, 0x2c, 0xf2, 0xb4, 0x19, 0x67, + 0x07, 0x3b, 0xd6, 0x01, 0xd8, 0xc4, 0x5b, 0xba, 0xf2, 0x9b, 0x7f, 0x70, 0xe1, 0x7d, 0xbf, 0xfd, + 0x07, 0x17, 0xde, 0xf7, 0xbb, 0x7f, 0x70, 0xe1, 0x7d, 0xdf, 0xf1, 0xe0, 0x82, 0xf5, 0x9b, 0x0f, + 0x2e, 0x58, 0xbf, 0xfd, 0xe0, 0x82, 0xf5, 0xbb, 0x0f, 0x2e, 0x58, 0xbf, 0xff, 0xe0, 0x82, 0xf5, + 0xa5, 0x3f, 0xbc, 0xf0, 0xbe, 0x4f, 0x15, 0xf6, 0x9e, 0xfb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, + 0x0f, 0xd0, 0xef, 0xf4, 0xac, 0x01, 0x01, 0x00, } func (m *AWSElasticBlockStoreVolumeSource) Marshal() (dAtA []byte, err error) { @@ -17949,6 +17951,37 @@ func (m *ServiceSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.IPFamilies) > 0 { + for iNdEx := len(m.IPFamilies) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.IPFamilies[iNdEx]) + copy(dAtA[i:], m.IPFamilies[iNdEx]) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.IPFamilies[iNdEx]))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x9a + } + } + if len(m.ClusterIPs) > 0 { + for iNdEx := len(m.ClusterIPs) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ClusterIPs[iNdEx]) + copy(dAtA[i:], m.ClusterIPs[iNdEx]) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.ClusterIPs[iNdEx]))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x92 + } + } + if m.IPFamilyPolicy != nil { + i -= len(*m.IPFamilyPolicy) + copy(dAtA[i:], *m.IPFamilyPolicy) + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.IPFamilyPolicy))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x8a + } if len(m.TopologyKeys) > 0 { for iNdEx := len(m.TopologyKeys) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.TopologyKeys[iNdEx]) @@ -17960,13 +17993,6 @@ func (m *ServiceSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x82 } } - if m.IPFamily != nil { - i -= len(*m.IPFamily) - copy(dAtA[i:], *m.IPFamily) - i = encodeVarintGenerated(dAtA, i, uint64(len(*m.IPFamily))) - i-- - dAtA[i] = 0x7a - } if m.SessionAffinityConfig != nil { { size, err := m.SessionAffinityConfig.MarshalToSizedBuffer(dAtA[:i]) @@ -23405,16 +23431,28 @@ func (m *ServiceSpec) Size() (n int) { l = m.SessionAffinityConfig.Size() n += 1 + l + sovGenerated(uint64(l)) } - if m.IPFamily != nil { - l = len(*m.IPFamily) - n += 1 + l + sovGenerated(uint64(l)) - } if len(m.TopologyKeys) > 0 { for _, s := range m.TopologyKeys { l = len(s) n += 2 + l + sovGenerated(uint64(l)) } } + if m.IPFamilyPolicy != nil { + l = len(*m.IPFamilyPolicy) + n += 2 + l + sovGenerated(uint64(l)) + } + if len(m.ClusterIPs) > 0 { + for _, s := range m.ClusterIPs { + l = len(s) + n += 2 + l + sovGenerated(uint64(l)) + } + } + if len(m.IPFamilies) > 0 { + for _, s := range m.IPFamilies { + l = len(s) + n += 2 + l + sovGenerated(uint64(l)) + } + } return n } @@ -26979,8 +27017,10 @@ func (this *ServiceSpec) String() string { `HealthCheckNodePort:` + fmt.Sprintf("%v", this.HealthCheckNodePort) + `,`, `PublishNotReadyAddresses:` + fmt.Sprintf("%v", this.PublishNotReadyAddresses) + `,`, `SessionAffinityConfig:` + strings.Replace(this.SessionAffinityConfig.String(), "SessionAffinityConfig", "SessionAffinityConfig", 1) + `,`, - `IPFamily:` + valueToStringGenerated(this.IPFamily) + `,`, `TopologyKeys:` + fmt.Sprintf("%v", this.TopologyKeys) + `,`, + `IPFamilyPolicy:` + valueToStringGenerated(this.IPFamilyPolicy) + `,`, + `ClusterIPs:` + fmt.Sprintf("%v", this.ClusterIPs) + `,`, + `IPFamilies:` + fmt.Sprintf("%v", this.IPFamilies) + `,`, `}`, }, "") return s @@ -63206,39 +63246,6 @@ func (m *ServiceSpec) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 15: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field IPFamily", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthGenerated - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := IPFamily(dAtA[iNdEx:postIndex]) - m.IPFamily = &s - iNdEx = postIndex case 16: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TopologyKeys", wireType) @@ -63271,6 +63278,103 @@ func (m *ServiceSpec) Unmarshal(dAtA []byte) error { } m.TopologyKeys = append(m.TopologyKeys, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 17: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field IPFamilyPolicy", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := IPFamilyPolicyType(dAtA[iNdEx:postIndex]) + m.IPFamilyPolicy = &s + iNdEx = postIndex + case 18: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterIPs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClusterIPs = append(m.ClusterIPs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 19: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field IPFamilies", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.IPFamilies = append(m.IPFamilies, IPFamily(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index 85dd3bf15af..8e19c949cff 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -4802,6 +4802,17 @@ message ServiceSpec { // +optional optional string clusterIP = 3; + // ClusterIPs identifies all the ClusterIPs assigned to this + // service. ClusterIPs are assigned or reserved based on the values of + // service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are + // allowed in ClusterIPs. The IPFamily of each ClusterIP must match + // values provided in service.spec.ipFamilies. Clients using ClusterIPs must + // keep it in sync with ClusterIP (if provided) by having ClusterIP matching + // first element of ClusterIPs. + // +listType=atomic + // +optional + repeated string clusterIPs = 18; + // type determines how the Service is exposed. Defaults to ClusterIP. Valid // options are ExternalName, ClusterIP, NodePort, and LoadBalancer. // "ExternalName" maps to the specified externalName. @@ -4889,23 +4900,15 @@ message ServiceSpec { // +optional optional SessionAffinityConfig sessionAffinityConfig = 14; - // ipFamily specifies whether this Service has a preference for a particular IP family (e.g. - // IPv4 vs. IPv6) when the IPv6DualStack feature gate is enabled. In a dual-stack cluster, - // you can specify ipFamily when creating a ClusterIP Service to determine whether the - // controller will allocate an IPv4 or IPv6 IP for it, and you can specify ipFamily when - // creating a headless Service to determine whether it will have IPv4 or IPv6 Endpoints. In - // either case, if you do not specify an ipFamily explicitly, it will default to the - // cluster's primary IP family. - // This field is part of an alpha feature, and you should not make any assumptions about its - // semantics other than those described above. In particular, you should not assume that it - // can (or cannot) be changed after creation time; that it can only have the values "IPv4" - // and "IPv6"; or that its current value on a given Service correctly reflects the current - // state of that Service. (For ClusterIP Services, look at clusterIP to see if the Service - // is IPv4 or IPv6. For headless Services, look at the endpoints, which may be dual-stack in - // the future. For ExternalName Services, ipFamily has no meaning, but it may be set to an - // irrelevant value anyway.) + // IPFamilies identifies all the IPFamilies assigned for this Service. If a value + // was not provided for IPFamilies it will be defaulted based on the cluster + // configuration and the value of service.spec.ipFamilyPolicy. A maximum of two + // values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is + // conditionally mutable: it allows for adding or removing a secondary IPFamily, + // but it does not allow changing the primary IPFamily of the service. + // +listType=atomic // +optional - optional string ipFamily = 15; + repeated string ipFamilies = 19; // topologyKeys is a preference-order list of topology keys which // implementations of services should use to preferentially sort endpoints @@ -4921,6 +4924,17 @@ message ServiceSpec { // If this is not specified or empty, no topology constraints will be applied. // +optional repeated string topologyKeys = 16; + + // IPFamilyPolicy represents the dual-stack-ness requested or required by this + // Service. If there is no value provided, then this Service will be considered + // SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), + // PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single + // IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies + // on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned + // to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs + // respectively. + // +optional + optional string ipFamilyPolicy = 17; } // ServiceStatus represents the current status of a service. diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 372fecfcb65..bd67349d957 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -3973,8 +3973,13 @@ type LoadBalancerIngress struct { Hostname string `json:"hostname,omitempty" protobuf:"bytes,2,opt,name=hostname"` } +const ( + // MaxServiceTopologyKeys is the largest number of topology keys allowed on a service + MaxServiceTopologyKeys = 16 +) + // IPFamily represents the IP Family (IPv4 or IPv6). This type is used -// to express the family of an IP expressed by a type (i.e. service.Spec.IPFamily) +// to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). type IPFamily string const ( @@ -3982,8 +3987,29 @@ const ( IPv4Protocol IPFamily = "IPv4" // IPv6Protocol indicates that this IP is IPv6 protocol IPv6Protocol IPFamily = "IPv6" - // MaxServiceTopologyKeys is the largest number of topology keys allowed on a service - MaxServiceTopologyKeys = 16 +) + +// IPFamilyPolicyType represents the dual-stack-ness requested or required by a Service +type IPFamilyPolicyType string + +const ( + // IPFamilyPolicySingleStack indicates that this service is required to have a single IPFamily. + // The IPFamily assigned is based on the default IPFamily used by the cluster + // or as identified by service.spec.ipFamilies field + IPFamilyPolicySingleStack IPFamilyPolicyType = "SingleStack" + // IPFamilyPolicyPreferDualStack indicates that this service prefers dual-stack when + // the cluster is configured for dual-stack. If the cluster is not configured + // for dual-stack the service will be assigned a single IPFamily. If the IPFamily is not + // set in service.spec.ipFamilies then the service will be assigned the default IPFamily + // configured on the cluster + IPFamilyPolicyPreferDualStack IPFamilyPolicyType = "PreferDualStack" + // IPFamilyPolicyRequireDualStack indicates that this service requires dual-stack. Using + // IPFamilyPolicyRequireDualStack on a single stack cluster will result in validation errors. The + // IPFamilies (and their order) assigned to this service is based on service.spec.ipFamilies. If + // service.spec.ipFamilies was not provided then it will be assigned according to how they are + // configured on the cluster. If service.spec.ipFamilies has only one entry then the alternative + // IPFamily will be added by apiserver + IPFamilyPolicyRequireDualStack IPFamilyPolicyType = "RequireDualStack" ) // ServiceSpec describes the attributes that a user creates on a service. @@ -4018,6 +4044,17 @@ type ServiceSpec struct { // +optional ClusterIP string `json:"clusterIP,omitempty" protobuf:"bytes,3,opt,name=clusterIP"` + // ClusterIPs identifies all the ClusterIPs assigned to this + // service. ClusterIPs are assigned or reserved based on the values of + // service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are + // allowed in ClusterIPs. The IPFamily of each ClusterIP must match + // values provided in service.spec.ipFamilies. Clients using ClusterIPs must + // keep it in sync with ClusterIP (if provided) by having ClusterIP matching + // first element of ClusterIPs. + // +listType=atomic + // +optional + ClusterIPs []string `json:"clusterIPs,omitempty" protobuf:"bytes,18,opt,name=clusterIPs"` + // type determines how the Service is exposed. Defaults to ClusterIP. Valid // options are ExternalName, ClusterIP, NodePort, and LoadBalancer. // "ExternalName" maps to the specified externalName. @@ -4105,23 +4142,15 @@ type ServiceSpec struct { // +optional SessionAffinityConfig *SessionAffinityConfig `json:"sessionAffinityConfig,omitempty" protobuf:"bytes,14,opt,name=sessionAffinityConfig"` - // ipFamily specifies whether this Service has a preference for a particular IP family (e.g. - // IPv4 vs. IPv6) when the IPv6DualStack feature gate is enabled. In a dual-stack cluster, - // you can specify ipFamily when creating a ClusterIP Service to determine whether the - // controller will allocate an IPv4 or IPv6 IP for it, and you can specify ipFamily when - // creating a headless Service to determine whether it will have IPv4 or IPv6 Endpoints. In - // either case, if you do not specify an ipFamily explicitly, it will default to the - // cluster's primary IP family. - // This field is part of an alpha feature, and you should not make any assumptions about its - // semantics other than those described above. In particular, you should not assume that it - // can (or cannot) be changed after creation time; that it can only have the values "IPv4" - // and "IPv6"; or that its current value on a given Service correctly reflects the current - // state of that Service. (For ClusterIP Services, look at clusterIP to see if the Service - // is IPv4 or IPv6. For headless Services, look at the endpoints, which may be dual-stack in - // the future. For ExternalName Services, ipFamily has no meaning, but it may be set to an - // irrelevant value anyway.) + // IPFamilies identifies all the IPFamilies assigned for this Service. If a value + // was not provided for IPFamilies it will be defaulted based on the cluster + // configuration and the value of service.spec.ipFamilyPolicy. A maximum of two + // values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is + // conditionally mutable: it allows for adding or removing a secondary IPFamily, + // but it does not allow changing the primary IPFamily of the service. + // +listType=atomic // +optional - IPFamily *IPFamily `json:"ipFamily,omitempty" protobuf:"bytes,15,opt,name=ipFamily,Configcasttype=IPFamily"` + IPFamilies []IPFamily `json:"ipFamilies,omitempty" protobuf:"bytes,19,opt,name=ipFamilies,casttype=IPFamily"` // topologyKeys is a preference-order list of topology keys which // implementations of services should use to preferentially sort endpoints @@ -4137,6 +4166,17 @@ type ServiceSpec struct { // If this is not specified or empty, no topology constraints will be applied. // +optional TopologyKeys []string `json:"topologyKeys,omitempty" protobuf:"bytes,16,opt,name=topologyKeys"` + + // IPFamilyPolicy represents the dual-stack-ness requested or required by this + // Service. If there is no value provided, then this Service will be considered + // SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), + // PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single + // IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies + // on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned + // to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs + // respectively. + // +optional + IPFamilyPolicy *IPFamilyPolicyType `json:"ipFamilyPolicy,omitempty" protobuf:"bytes,17,opt,name=ipFamilyPolicy,casttype=IPFamilyPolicyType"` } // ServicePort contains information on service's port. diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index 7a1cd775cc9..a71dc118f24 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -2230,6 +2230,7 @@ var map_ServiceSpec = map[string]string{ "ports": "The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "selector": "Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/", "clusterIP": "clusterIP is the IP address of the service and is usually assigned randomly by the master. If an address is specified manually and is not in use by others, it will be allocated to the service; otherwise, creation of the service will fail. This field can not be changed through updates. Valid values are \"None\", empty string (\"\"), or a valid IP address. \"None\" can be specified for headless services when proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", + "clusterIPs": "ClusterIPs identifies all the ClusterIPs assigned to this service. ClusterIPs are assigned or reserved based on the values of service.spec.ipFamilies. A maximum of two entries (dual-stack IPs) are allowed in ClusterIPs. The IPFamily of each ClusterIP must match values provided in service.spec.ipFamilies. Clients using ClusterIPs must keep it in sync with ClusterIP (if provided) by having ClusterIP matching first element of ClusterIPs.", "type": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", "externalIPs": "externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system.", "sessionAffinity": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", @@ -2240,8 +2241,9 @@ var map_ServiceSpec = map[string]string{ "healthCheckNodePort": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "publishNotReadyAddresses": "publishNotReadyAddresses indicates that any agent which deals with endpoints for this Service should disregard any indications of ready/not-ready. The primary use case for setting this field is for a StatefulSet's Headless Service to propagate SRV DNS records for its Pods for the purpose of peer discovery. The Kubernetes controllers that generate Endpoints and EndpointSlice resources for Services interpret this to mean that all endpoints are considered \"ready\" even if the Pods themselves are not. Agents which consume only Kubernetes generated endpoints through the Endpoints or EndpointSlice resources can safely assume this behavior.", "sessionAffinityConfig": "sessionAffinityConfig contains the configurations of session affinity.", - "ipFamily": "ipFamily specifies whether this Service has a preference for a particular IP family (e.g. IPv4 vs. IPv6) when the IPv6DualStack feature gate is enabled. In a dual-stack cluster, you can specify ipFamily when creating a ClusterIP Service to determine whether the controller will allocate an IPv4 or IPv6 IP for it, and you can specify ipFamily when creating a headless Service to determine whether it will have IPv4 or IPv6 Endpoints. In either case, if you do not specify an ipFamily explicitly, it will default to the cluster's primary IP family. This field is part of an alpha feature, and you should not make any assumptions about its semantics other than those described above. In particular, you should not assume that it can (or cannot) be changed after creation time; that it can only have the values \"IPv4\" and \"IPv6\"; or that its current value on a given Service correctly reflects the current state of that Service. (For ClusterIP Services, look at clusterIP to see if the Service is IPv4 or IPv6. For headless Services, look at the endpoints, which may be dual-stack in the future. For ExternalName Services, ipFamily has no meaning, but it may be set to an irrelevant value anyway.)", + "ipFamilies": "IPFamilies identifies all the IPFamilies assigned for this Service. If a value was not provided for IPFamilies it will be defaulted based on the cluster configuration and the value of service.spec.ipFamilyPolicy. A maximum of two values (dual-stack IPFamilies) are allowed in IPFamilies. IPFamilies field is conditionally mutable: it allows for adding or removing a secondary IPFamily, but it does not allow changing the primary IPFamily of the service.", "topologyKeys": "topologyKeys is a preference-order list of topology keys which implementations of services should use to preferentially sort endpoints when accessing this Service, it can not be used at the same time as externalTrafficPolicy=Local. Topology keys must be valid label keys and at most 16 keys may be specified. Endpoints are chosen based on the first topology key with available backends. If this field is specified and all entries have no backends that match the topology of the client, the service has no backends for that client and connections should fail. The special value \"*\" may be used to mean \"any topology\". This catch-all value, if used, only makes sense as the last value in the list. If this is not specified or empty, no topology constraints will be applied.", + "ipFamilyPolicy": "IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this Service will be considered SingleStack (single IPFamily). Services can be SingleStack (single IPFamily), PreferDualStack (two dual-stack IPFamilies on dual-stack clusters or single IPFamily on single-stack clusters), or RequireDualStack (two dual-stack IPFamilies on dual-stack configured clusters, otherwise fail). The IPFamilies and ClusterIPs assigned to this service can be controlled by service.spec.ipFamilies and service.spec.clusterIPs respectively.", } func (ServiceSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go index 445c7c04a9d..2b070041496 100644 --- a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go @@ -5270,6 +5270,11 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { (*out)[key] = val } } + if in.ClusterIPs != nil { + in, out := &in.ClusterIPs, &out.ClusterIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.ExternalIPs != nil { in, out := &in.ExternalIPs, &out.ExternalIPs *out = make([]string, len(*in)) @@ -5285,16 +5290,21 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = new(SessionAffinityConfig) (*in).DeepCopyInto(*out) } - if in.IPFamily != nil { - in, out := &in.IPFamily, &out.IPFamily - *out = new(IPFamily) - **out = **in + if in.IPFamilies != nil { + in, out := &in.IPFamilies, &out.IPFamilies + *out = make([]IPFamily, len(*in)) + copy(*out, *in) } if in.TopologyKeys != nil { in, out := &in.TopologyKeys, &out.TopologyKeys *out = make([]string, len(*in)) copy(*out, *in) } + if in.IPFamilyPolicy != nil { + in, out := &in.IPFamilyPolicy, &out.IPFamilyPolicy + *out = new(IPFamilyPolicyType) + **out = **in + } return } diff --git a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json index 4a3e2caf824..338d9936fab 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json +++ b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json @@ -55,35 +55,41 @@ "22": "23" }, "clusterIP": "24", - "type": ".蘯6ċV夸", - "externalIPs": [ + "clusterIPs": [ "25" ], - "sessionAffinity": "ɑ", - "loadBalancerIP": "26", - "loadBalancerSourceRanges": [ - "27" + "type": "鮽ort昍řČ扷5ƗǸƢ6/ʕVŚ(Ŀ", + "externalIPs": [ + "26" ], - "externalName": "28", - "externalTrafficPolicy": "ʤ脽ěĂ凗蓏Ŋ蛊ĉy緅縕\u003eŽ", - "healthCheckNodePort": -1095807277, + "sessionAffinity": "甞谐颋DžS", + "loadBalancerIP": "27", + "loadBalancerSourceRanges": [ + "28" + ], + "externalName": "29", + "externalTrafficPolicy": "ƏS$+½H牗洝尿", + "healthCheckNodePort": -1965738697, "publishNotReadyAddresses": true, "sessionAffinityConfig": { "clientIP": { - "timeoutSeconds": -1973740160 + "timeoutSeconds": 2072604405 } }, - "ipFamily": "³-Ǐ忄*齧獚敆ȎțêɘIJ斬", + "ipFamilies": [ + "x" + ], "topologyKeys": [ - "29" - ] + "30" + ], + "ipFamilyPolicy": ";Ơ歿:狞夌碕ʂ" }, "status": { "loadBalancer": { "ingress": [ { - "ip": "30", - "hostname": "31" + "ip": "31", + "hostname": "32" } ] } diff --git a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.pb b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.pb index 223cbb9b9ac..3993d0a896e 100644 Binary files a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.pb and b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.pb differ diff --git a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.yaml b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.yaml index d80d33427fb..0bb62ed52e0 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.yaml @@ -31,15 +31,19 @@ metadata: uid: "7" spec: clusterIP: "24" - externalIPs: + clusterIPs: - "25" - externalName: "28" - externalTrafficPolicy: ʤ脽ěĂ凗蓏Ŋ蛊ĉy緅縕>Ž - healthCheckNodePort: -1095807277 - ipFamily: ³-Ǐ忄*齧獚敆ȎțêɘIJ斬 - loadBalancerIP: "26" + externalIPs: + - "26" + externalName: "29" + externalTrafficPolicy: ƏS$+½H牗洝尿 + healthCheckNodePort: -1965738697 + ipFamilies: + - x + ipFamilyPolicy: ;Ơ歿:狞夌碕ʂ + loadBalancerIP: "27" loadBalancerSourceRanges: - - "27" + - "28" ports: - appProtocol: "20" name: "19" @@ -50,15 +54,15 @@ spec: publishNotReadyAddresses: true selector: "22": "23" - sessionAffinity: ɑ + sessionAffinity: 甞谐颋DžS sessionAffinityConfig: clientIP: - timeoutSeconds: -1973740160 + timeoutSeconds: 2072604405 topologyKeys: - - "29" - type: .蘯6ċV夸 + - "30" + type: 鮽ort昍řČ扷5ƗǸƢ6/ʕVŚ(Ŀ status: loadBalancer: ingress: - - hostname: "31" - ip: "30" + - hostname: "32" + ip: "31" diff --git a/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.json b/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.json new file mode 100644 index 00000000000..b9ce94f3941 --- /dev/null +++ b/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.json @@ -0,0 +1,90 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "2", + "generateName": "3", + "namespace": "4", + "selfLink": "5", + "uid": "7", + "resourceVersion": "11042405498087606203", + "generation": 8071137005907523419, + "creationTimestamp": null, + "deletionGracePeriodSeconds": -4955867275792137171, + "labels": { + "7": "8" + }, + "annotations": { + "9": "10" + }, + "ownerReferences": [ + { + "apiVersion": "11", + "kind": "12", + "name": "13", + "uid": "Dz廔ȇ{sŊƏp", + "controller": false, + "blockOwnerDeletion": true + } + ], + "finalizers": [ + "14" + ], + "clusterName": "15", + "managedFields": [ + { + "manager": "16", + "operation": "鐊唊飙Ş-U圴÷a/ɔ}摁(湗Ć]", + "apiVersion": "17", + "fieldsType": "18" + } + ] + }, + "spec": { + "ports": [ + { + "name": "19", + "protocol": "@Hr鯹)晿", + "appProtocol": "20", + "port": 202283346, + "targetPort": "21", + "nodePort": -474380055 + } + ], + "selector": { + "22": "23" + }, + "clusterIP": "24", + "type": ".蘯6ċV夸", + "externalIPs": [ + "25" + ], + "sessionAffinity": "ɑ", + "loadBalancerIP": "26", + "loadBalancerSourceRanges": [ + "27" + ], + "externalName": "28", + "externalTrafficPolicy": "ʤ脽ěĂ凗蓏Ŋ蛊ĉy緅縕\u003eŽ", + "healthCheckNodePort": -1095807277, + "publishNotReadyAddresses": true, + "sessionAffinityConfig": { + "clientIP": { + "timeoutSeconds": -1973740160 + } + }, + "topologyKeys": [ + "29" + ] + }, + "status": { + "loadBalancer": { + "ingress": [ + { + "ip": "30", + "hostname": "31" + } + ] + } + } +} \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.pb b/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.pb new file mode 100644 index 00000000000..8bc7d5d61a8 Binary files /dev/null and b/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.pb differ diff --git a/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.yaml b/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.yaml new file mode 100644 index 00000000000..45117fd8ee4 --- /dev/null +++ b/staging/src/k8s.io/api/testdata/v1.18.0/core.v1.Service.after_roundtrip.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + "9": "10" + clusterName: "15" + creationTimestamp: null + deletionGracePeriodSeconds: -4955867275792137171 + finalizers: + - "14" + generateName: "3" + generation: 8071137005907523419 + labels: + "7": "8" + managedFields: + - apiVersion: "17" + fieldsType: "18" + manager: "16" + operation: 鐊唊飙Ş-U圴÷a/ɔ}摁(湗Ć] + name: "2" + namespace: "4" + ownerReferences: + - apiVersion: "11" + blockOwnerDeletion: true + controller: false + kind: "12" + name: "13" + uid: Dz廔ȇ{sŊƏp + resourceVersion: "11042405498087606203" + selfLink: "5" + uid: "7" +spec: + clusterIP: "24" + externalIPs: + - "25" + externalName: "28" + externalTrafficPolicy: ʤ脽ěĂ凗蓏Ŋ蛊ĉy緅縕>Ž + healthCheckNodePort: -1095807277 + loadBalancerIP: "26" + loadBalancerSourceRanges: + - "27" + ports: + - appProtocol: "20" + name: "19" + nodePort: -474380055 + port: 202283346 + protocol: '@Hr鯹)晿' + targetPort: "21" + publishNotReadyAddresses: true + selector: + "22": "23" + sessionAffinity: ɑ + sessionAffinityConfig: + clientIP: + timeoutSeconds: -1973740160 + topologyKeys: + - "29" + type: .蘯6ċV夸 +status: + loadBalancer: + ingress: + - hostname: "31" + ip: "30" diff --git a/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.json b/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.json new file mode 100644 index 00000000000..b9ce94f3941 --- /dev/null +++ b/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.json @@ -0,0 +1,90 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "2", + "generateName": "3", + "namespace": "4", + "selfLink": "5", + "uid": "7", + "resourceVersion": "11042405498087606203", + "generation": 8071137005907523419, + "creationTimestamp": null, + "deletionGracePeriodSeconds": -4955867275792137171, + "labels": { + "7": "8" + }, + "annotations": { + "9": "10" + }, + "ownerReferences": [ + { + "apiVersion": "11", + "kind": "12", + "name": "13", + "uid": "Dz廔ȇ{sŊƏp", + "controller": false, + "blockOwnerDeletion": true + } + ], + "finalizers": [ + "14" + ], + "clusterName": "15", + "managedFields": [ + { + "manager": "16", + "operation": "鐊唊飙Ş-U圴÷a/ɔ}摁(湗Ć]", + "apiVersion": "17", + "fieldsType": "18" + } + ] + }, + "spec": { + "ports": [ + { + "name": "19", + "protocol": "@Hr鯹)晿", + "appProtocol": "20", + "port": 202283346, + "targetPort": "21", + "nodePort": -474380055 + } + ], + "selector": { + "22": "23" + }, + "clusterIP": "24", + "type": ".蘯6ċV夸", + "externalIPs": [ + "25" + ], + "sessionAffinity": "ɑ", + "loadBalancerIP": "26", + "loadBalancerSourceRanges": [ + "27" + ], + "externalName": "28", + "externalTrafficPolicy": "ʤ脽ěĂ凗蓏Ŋ蛊ĉy緅縕\u003eŽ", + "healthCheckNodePort": -1095807277, + "publishNotReadyAddresses": true, + "sessionAffinityConfig": { + "clientIP": { + "timeoutSeconds": -1973740160 + } + }, + "topologyKeys": [ + "29" + ] + }, + "status": { + "loadBalancer": { + "ingress": [ + { + "ip": "30", + "hostname": "31" + } + ] + } + } +} \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.pb b/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.pb new file mode 100644 index 00000000000..8bc7d5d61a8 Binary files /dev/null and b/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.pb differ diff --git a/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.yaml b/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.yaml new file mode 100644 index 00000000000..45117fd8ee4 --- /dev/null +++ b/staging/src/k8s.io/api/testdata/v1.19.0/core.v1.Service.after_roundtrip.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + "9": "10" + clusterName: "15" + creationTimestamp: null + deletionGracePeriodSeconds: -4955867275792137171 + finalizers: + - "14" + generateName: "3" + generation: 8071137005907523419 + labels: + "7": "8" + managedFields: + - apiVersion: "17" + fieldsType: "18" + manager: "16" + operation: 鐊唊飙Ş-U圴÷a/ɔ}摁(湗Ć] + name: "2" + namespace: "4" + ownerReferences: + - apiVersion: "11" + blockOwnerDeletion: true + controller: false + kind: "12" + name: "13" + uid: Dz廔ȇ{sŊƏp + resourceVersion: "11042405498087606203" + selfLink: "5" + uid: "7" +spec: + clusterIP: "24" + externalIPs: + - "25" + externalName: "28" + externalTrafficPolicy: ʤ脽ěĂ凗蓏Ŋ蛊ĉy緅縕>Ž + healthCheckNodePort: -1095807277 + loadBalancerIP: "26" + loadBalancerSourceRanges: + - "27" + ports: + - appProtocol: "20" + name: "19" + nodePort: -474380055 + port: 202283346 + protocol: '@Hr鯹)晿' + targetPort: "21" + publishNotReadyAddresses: true + selector: + "22": "23" + sessionAffinity: ɑ + sessionAffinityConfig: + clientIP: + timeoutSeconds: -1973740160 + topologyKeys: + - "29" + type: .蘯6ċV夸 +status: + loadBalancer: + ingress: + - hostname: "31" + ip: "30" diff --git a/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go b/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go index f6268ec49cf..c8b41998405 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go @@ -347,7 +347,7 @@ func IsValidPortName(port string) []string { // IsValidIP tests that the argument is a valid IP address. func IsValidIP(value string) []string { if net.ParseIP(value) == nil { - return []string{"must be a valid IP address, (e.g. 10.9.8.7)"} + return []string{"must be a valid IP address, (e.g. 10.9.8.7 or 2001:db8::ffff)"} } return nil } diff --git a/staging/src/k8s.io/kubectl/pkg/describe/describe.go b/staging/src/k8s.io/kubectl/pkg/describe/describe.go index 41cbd2e7de9..0621c590db5 100644 --- a/staging/src/k8s.io/kubectl/pkg/describe/describe.go +++ b/staging/src/k8s.io/kubectl/pkg/describe/describe.go @@ -2721,10 +2721,27 @@ func describeService(service *corev1.Service, endpoints *corev1.Endpoints, event printAnnotationsMultiline(w, "Annotations", service.Annotations) w.Write(LEVEL_0, "Selector:\t%s\n", labels.FormatLabels(service.Spec.Selector)) w.Write(LEVEL_0, "Type:\t%s\n", service.Spec.Type) - w.Write(LEVEL_0, "IP:\t%s\n", service.Spec.ClusterIP) - if service.Spec.IPFamily != nil { - w.Write(LEVEL_0, "IPFamily:\t%s\n", *(service.Spec.IPFamily)) + if service.Spec.IPFamilyPolicy != nil { + w.Write(LEVEL_0, "IP Family Policy:\t%s\n", *(service.Spec.IPFamilyPolicy)) + } + + if len(service.Spec.IPFamilies) > 0 { + ipfamiliesStrings := make([]string, 0, len(service.Spec.IPFamilies)) + for _, family := range service.Spec.IPFamilies { + ipfamiliesStrings = append(ipfamiliesStrings, string(family)) + } + + w.Write(LEVEL_0, "IP Families:\t%s\n", strings.Join(ipfamiliesStrings, ",")) + } else { + w.Write(LEVEL_0, "IP Families:\t%s\n", "") + } + + w.Write(LEVEL_0, "IP:\t%s\n", service.Spec.ClusterIP) + if len(service.Spec.ClusterIPs) > 0 { + w.Write(LEVEL_0, "IPs:\t%s\n", strings.Join(service.Spec.ClusterIPs, ",")) + } else { + w.Write(LEVEL_0, "IPs:\t%s\n", "") } if len(service.Spec.ExternalIPs) > 0 { diff --git a/staging/src/k8s.io/kubectl/pkg/describe/describe_test.go b/staging/src/k8s.io/kubectl/pkg/describe/describe_test.go index 8b9fdd8223d..42fcb47b965 100644 --- a/staging/src/k8s.io/kubectl/pkg/describe/describe_test.go +++ b/staging/src/k8s.io/kubectl/pkg/describe/describe_test.go @@ -358,8 +358,7 @@ func getResourceList(cpu, memory string) corev1.ResourceList { } func TestDescribeService(t *testing.T) { - defaultServiceIPFamily := corev1.IPv4Protocol - + singleStack := corev1.IPFamilyPolicySingleStack testCases := []struct { name string service *corev1.Service @@ -373,8 +372,7 @@ func TestDescribeService(t *testing.T) { Namespace: "foo", }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - IPFamily: &defaultServiceIPFamily, + Type: corev1.ServiceTypeLoadBalancer, Ports: []corev1.ServicePort{{ Name: "port-tcp", Port: 8080, @@ -384,6 +382,7 @@ func TestDescribeService(t *testing.T) { }}, Selector: map[string]string{"blah": "heh"}, ClusterIP: "1.2.3.4", + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, LoadBalancerIP: "5.6.7.8", SessionAffinity: "None", ExternalTrafficPolicy: "Local", @@ -412,8 +411,7 @@ func TestDescribeService(t *testing.T) { Namespace: "foo", }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - IPFamily: &defaultServiceIPFamily, + Type: corev1.ServiceTypeLoadBalancer, Ports: []corev1.ServicePort{{ Name: "port-tcp", Port: 8080, @@ -423,6 +421,7 @@ func TestDescribeService(t *testing.T) { }}, Selector: map[string]string{"blah": "heh"}, ClusterIP: "1.2.3.4", + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, LoadBalancerIP: "5.6.7.8", SessionAffinity: "None", ExternalTrafficPolicy: "Local", @@ -451,8 +450,7 @@ func TestDescribeService(t *testing.T) { Namespace: "foo", }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - IPFamily: &defaultServiceIPFamily, + Type: corev1.ServiceTypeLoadBalancer, Ports: []corev1.ServicePort{{ Name: "port-tcp", Port: 8080, @@ -462,6 +460,7 @@ func TestDescribeService(t *testing.T) { }}, Selector: map[string]string{"blah": "heh"}, ClusterIP: "1.2.3.4", + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, LoadBalancerIP: "5.6.7.8", SessionAffinity: "None", ExternalTrafficPolicy: "Local", @@ -474,7 +473,51 @@ func TestDescribeService(t *testing.T) { "Selector", "blah=heh", "Type", "LoadBalancer", "IP", "1.2.3.4", - "IPFamily", "IPv4", + "IP Families", "IPv4", + "Port", "port-tcp", "8080/TCP", + "TargetPort", "targetPort/TCP", + "NodePort", "port-tcp", "31111/TCP", + "Session Affinity", "None", + "External Traffic Policy", "Local", + "HealthCheck NodePort", "32222", + }, + }, + { + name: "test-ServiceIPFamilyPolicy+ClusterIPs", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "foo", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{{ + Name: "port-tcp", + Port: 8080, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("targetPort"), + NodePort: 31111, + }}, + Selector: map[string]string{"blah": "heh"}, + ClusterIP: "1.2.3.4", + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + IPFamilyPolicy: &singleStack, + ClusterIPs: []string{"1.2.3.4"}, + LoadBalancerIP: "5.6.7.8", + SessionAffinity: "None", + ExternalTrafficPolicy: "Local", + HealthCheckNodePort: 32222, + }, + }, + expect: []string{ + "Name", "bar", + "Namespace", "foo", + "Selector", "blah=heh", + "Type", "LoadBalancer", + "IP", "1.2.3.4", + "IP Families", "IPv4", + "IP Family Policy", "SingleStack", + "IPs", "1.2.3.4", "Port", "port-tcp", "8080/TCP", "TargetPort", "targetPort/TCP", "NodePort", "port-tcp", "31111/TCP", diff --git a/test/cmd/apply.sh b/test/cmd/apply.sh index 34c42b6af77..e3e2d74184f 100755 --- a/test/cmd/apply.sh +++ b/test/cmd/apply.sh @@ -269,7 +269,7 @@ __EOF__ kube::test::get_object_assert 'services a' "{{${id_field:?}}}" 'a' # change immutable field and apply service a output_message=$(! kubectl apply -f hack/testdata/service-revision2.yaml 2>&1 "${kube_flags[@]:?}") - kube::test::if_has_string "${output_message}" 'field is immutable' + kube::test::if_has_string "${output_message}" 'may not change once set' # apply --force to recreate resources for immutable fields kubectl apply -f hack/testdata/service-revision2.yaml --force "${kube_flags[@]:?}" # check immutable field exists diff --git a/test/e2e/network/dual_stack.go b/test/e2e/network/dual_stack.go index 6feb9aa9512..976fbd12196 100644 --- a/test/e2e/network/dual_stack.go +++ b/test/e2e/network/dual_stack.go @@ -212,16 +212,11 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { assertNetworkConnectivity(f, *serverPods, *clientPods, "dualstack-test-client", "80") }) - ginkgo.It("should create service with cluster ip from primary service range [Feature:IPv6DualStackAlphaFeature:Phase2]", func() { + ginkgo.It("should create a single stack service with cluster ip from primary service range [Feature:IPv6DualStackAlphaFeature:Phase2]", func() { serviceName := "defaultclusterip" ns := f.Namespace.Name jig := e2eservice.NewTestJig(cs, ns, serviceName) - defaultIPFamily := v1.IPv4Protocol - if framework.TestContext.ClusterIsIPv6() { - defaultIPFamily = v1.IPv6Protocol - } - t := NewServerTest(cs, ns, serviceName) defer func() { defer ginkgo.GinkgoRecover() @@ -230,8 +225,8 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { } }() - ginkgo.By("creating service " + ns + "/" + serviceName + " with Service.Spec.IPFamily not set") - service := createService(t.ServiceName, t.Namespace, t.Labels, nil) + ginkgo.By("creating service " + ns + "/" + serviceName + " with Service.Spec.IPFamilies not set nil policy") + service := createService(t.ServiceName, t.Namespace, t.Labels, nil, nil) jig.Labels = t.Labels err := jig.CreateServicePods(2) @@ -241,8 +236,14 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { validateNumOfServicePorts(svc, 2) + expectedPolicy := v1.IPFamilyPolicySingleStack + expectedFamilies := []v1.IPFamily{v1.IPv4Protocol} + if framework.TestContext.ClusterIsIPv6() { + expectedFamilies = []v1.IPFamily{v1.IPv6Protocol} + } + // check the spec has been set to default ip family - validateServiceAndClusterIPFamily(svc, defaultIPFamily) + validateServiceAndClusterIPFamily(svc, expectedFamilies, &expectedPolicy) // ensure endpoint belong to same ipfamily as service if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { @@ -250,7 +251,8 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { if err != nil { return false, nil } - validateEndpointsBelongToIPFamily(svc, endpoint, defaultIPFamily) + validateEndpointsBelongToIPFamily(svc, endpoint, expectedFamilies[0] /*endpoint controller works on primary ip*/) + return true, nil }); err != nil { framework.Failf("Get endpoints for service %s/%s failed (%s)", svc.Namespace, svc.Name, err) @@ -260,7 +262,6 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { ginkgo.It("should create service with ipv4 cluster ip [Feature:IPv6DualStackAlphaFeature:Phase2]", func() { serviceName := "ipv4clusterip" ns := f.Namespace.Name - ipv4 := v1.IPv4Protocol jig := e2eservice.NewTestJig(cs, ns, serviceName) @@ -273,7 +274,11 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { }() ginkgo.By("creating service " + ns + "/" + serviceName + " with Service.Spec.IPFamily IPv4" + ns) - service := createService(t.ServiceName, t.Namespace, t.Labels, &ipv4) + + expectedPolicy := v1.IPFamilyPolicySingleStack + expectedFamilies := []v1.IPFamily{v1.IPv4Protocol} + + service := createService(t.ServiceName, t.Namespace, t.Labels, nil, expectedFamilies) jig.Labels = t.Labels err := jig.CreateServicePods(2) @@ -284,7 +289,7 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { validateNumOfServicePorts(svc, 2) // check the spec has been set to IPv4 and cluster ip belong to IPv4 family - validateServiceAndClusterIPFamily(svc, ipv4) + validateServiceAndClusterIPFamily(svc, expectedFamilies, &expectedPolicy) // ensure endpoints belong to same ipfamily as service if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { @@ -292,7 +297,7 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { if err != nil { return false, nil } - validateEndpointsBelongToIPFamily(svc, endpoint, ipv4) + validateEndpointsBelongToIPFamily(svc, endpoint, expectedFamilies[0] /* endpoint controller operates on primary ip */) return true, nil }); err != nil { framework.Failf("Get endpoints for service %s/%s failed (%s)", svc.Namespace, svc.Name, err) @@ -315,7 +320,10 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { }() ginkgo.By("creating service " + ns + "/" + serviceName + " with Service.Spec.IPFamily IPv6" + ns) - service := createService(t.ServiceName, t.Namespace, t.Labels, &ipv6) + expectedPolicy := v1.IPFamilyPolicySingleStack + expectedFamilies := []v1.IPFamily{v1.IPv6Protocol} + + service := createService(t.ServiceName, t.Namespace, t.Labels, nil, expectedFamilies) jig.Labels = t.Labels err := jig.CreateServicePods(2) @@ -326,7 +334,7 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { validateNumOfServicePorts(svc, 2) // check the spec has been set to IPv6 and cluster ip belongs to IPv6 family - validateServiceAndClusterIPFamily(svc, ipv6) + validateServiceAndClusterIPFamily(svc, expectedFamilies, &expectedPolicy) // ensure endpoints belong to same ipfamily as service if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { @@ -340,6 +348,98 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() { framework.Failf("Get endpoints for service %s/%s failed (%s)", svc.Namespace, svc.Name, err) } }) + + ginkgo.It("should create service with ipv4,v6 cluster ip [Feature:IPv6DualStackAlphaFeature:Phase2]", func() { + serviceName := "ipv4ipv6clusterip" + ns := f.Namespace.Name + + jig := e2eservice.NewTestJig(cs, ns, serviceName) + + t := NewServerTest(cs, ns, serviceName) + defer func() { + defer ginkgo.GinkgoRecover() + if errs := t.Cleanup(); len(errs) != 0 { + framework.Failf("errors in cleanup: %v", errs) + } + }() + + ginkgo.By("creating service " + ns + "/" + serviceName + " with Service.Spec.IPFamily IPv4, IPv6" + ns) + + expectedPolicy := v1.IPFamilyPolicyRequireDualStack + expectedFamilies := []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} + + service := createService(t.ServiceName, t.Namespace, t.Labels, nil, expectedFamilies) + + jig.Labels = t.Labels + err := jig.CreateServicePods(2) + framework.ExpectNoError(err) + svc, err := t.CreateService(service) + framework.ExpectNoError(err, "failed to create service: %s in namespace: %s", serviceName, ns) + + validateNumOfServicePorts(svc, 2) + + // check the spec has been set to IPv4 and cluster ip belong to IPv4 family + validateServiceAndClusterIPFamily(svc, expectedFamilies, &expectedPolicy) + + // ensure endpoints belong to same ipfamily as service + if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { + endpoint, err := cs.CoreV1().Endpoints(svc.Namespace).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + validateEndpointsBelongToIPFamily(svc, endpoint, expectedFamilies[0] /* endpoint controller operates on primary ip */) + return true, nil + }); err != nil { + framework.Failf("Get endpoints for service %s/%s failed (%s)", svc.Namespace, svc.Name, err) + } + }) + + ginkgo.It("should create service with ipv6,v4 cluster ip [Feature:IPv6DualStackAlphaFeature:Phase2]", func() { + serviceName := "ipv6ipv4clusterip" + ns := f.Namespace.Name + + jig := e2eservice.NewTestJig(cs, ns, serviceName) + + t := NewServerTest(cs, ns, serviceName) + defer func() { + defer ginkgo.GinkgoRecover() + if errs := t.Cleanup(); len(errs) != 0 { + framework.Failf("errors in cleanup: %v", errs) + } + }() + + ginkgo.By("creating service " + ns + "/" + serviceName + " with Service.Spec.IPFamily IPv4, IPv6" + ns) + + expectedPolicy := v1.IPFamilyPolicyRequireDualStack + expectedFamilies := []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol} + + service := createService(t.ServiceName, t.Namespace, t.Labels, nil, expectedFamilies) + + jig.Labels = t.Labels + err := jig.CreateServicePods(2) + framework.ExpectNoError(err) + svc, err := t.CreateService(service) + framework.ExpectNoError(err, "failed to create service: %s in namespace: %s", serviceName, ns) + + validateNumOfServicePorts(svc, 2) + + // check the spec has been set to IPv4 and cluster ip belong to IPv4 family + validateServiceAndClusterIPFamily(svc, expectedFamilies, &expectedPolicy) + + // ensure endpoints belong to same ipfamily as service + if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { + endpoint, err := cs.CoreV1().Endpoints(svc.Namespace).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + validateEndpointsBelongToIPFamily(svc, endpoint, expectedFamilies[0] /* endpoint controller operates on primary ip */) + return true, nil + }); err != nil { + framework.Failf("Get endpoints for service %s/%s failed (%s)", svc.Namespace, svc.Name, err) + } + }) + // TODO (khenidak add slice validation logic, since endpoint controller only operates + // on primary ClusterIP }) func validateNumOfServicePorts(svc *v1.Service, expectedNumOfPorts int) { @@ -348,17 +448,37 @@ func validateNumOfServicePorts(svc *v1.Service, expectedNumOfPorts int) { } } -func validateServiceAndClusterIPFamily(svc *v1.Service, expectedIPFamily v1.IPFamily) { - if svc.Spec.IPFamily == nil { +func validateServiceAndClusterIPFamily(svc *v1.Service, expectedIPFamilies []v1.IPFamily, expectedPolicy *v1.IPFamilyPolicyType) { + if len(svc.Spec.IPFamilies) != len(expectedIPFamilies) { framework.Failf("service ip family nil for service %s/%s", svc.Namespace, svc.Name) } - if *svc.Spec.IPFamily != expectedIPFamily { - framework.Failf("ip family mismatch for service: %s/%s, expected: %s, actual: %s", svc.Namespace, svc.Name, expectedIPFamily, *svc.Spec.IPFamily) + + for idx, family := range expectedIPFamilies { + if svc.Spec.IPFamilies[idx] != family { + framework.Failf("service %s/%s expected family %v at index[%v] got %v", svc.Namespace, svc.Name, family, idx, svc.Spec.IPFamilies[idx]) + } } - isIPv6ClusterIP := netutils.IsIPv6String(svc.Spec.ClusterIP) - if (expectedIPFamily == v1.IPv4Protocol && isIPv6ClusterIP) || (expectedIPFamily == v1.IPv6Protocol && !isIPv6ClusterIP) { - framework.Failf("got unexpected service ip %s, should belong to %s ip family", svc.Spec.ClusterIP, expectedIPFamily) + // validate ip assigned is from the family + if len(svc.Spec.ClusterIPs) != len(svc.Spec.IPFamilies) { + framework.Failf("service %s/%s assigned ips [%+v] does not match families [%+v]", svc.Namespace, svc.Name, svc.Spec.ClusterIPs, svc.Spec.IPFamilies) + } + + for idx, family := range svc.Spec.IPFamilies { + if (family == v1.IPv6Protocol) != netutils.IsIPv6String(svc.Spec.ClusterIPs[idx]) { + framework.Failf("service %s/%s assigned ips at [%v]:%v does not match family:%v", svc.Namespace, svc.Name, idx, svc.Spec.ClusterIPs[idx], family) + } + } + // validate policy + if expectedPolicy == nil && svc.Spec.IPFamilyPolicy != nil { + framework.Failf("service %s/%s expected nil for IPFamilyPolicy", svc.Namespace, svc.Name) + } + if expectedPolicy != nil && svc.Spec.IPFamilyPolicy == nil { + framework.Failf("service %s/%s expected value %v for IPFamilyPolicy", svc.Namespace, svc.Name, expectedPolicy) + } + + if expectedPolicy != nil && *(svc.Spec.IPFamilyPolicy) != *(expectedPolicy) { + framework.Failf("service %s/%s expected value %v for IPFamilyPolicy", svc.Namespace, svc.Name, expectedPolicy) } } @@ -368,7 +488,7 @@ func validateEndpointsBelongToIPFamily(svc *v1.Service, endpoint *v1.Endpoints, } for _, ss := range endpoint.Subsets { for _, e := range ss.Addresses { - if (expectedIPFamily == v1.IPv6Protocol && isIPv4(e.IP)) || (expectedIPFamily == v1.IPv4Protocol && netutils.IsIPv6String(e.IP)) { + if (expectedIPFamily == v1.IPv6Protocol) != netutils.IsIPv6String(e.IP) { framework.Failf("service endpoint %s doesn't belong to %s ip family", e.IP, expectedIPFamily) } } @@ -427,16 +547,17 @@ func isIPv4CIDR(cidr string) bool { } // createService returns a service spec with defined arguments -func createService(name, ns string, labels map[string]string, ipFamily *v1.IPFamily) *v1.Service { +func createService(name, ns string, labels map[string]string, ipFamilyPolicy *v1.IPFamilyPolicyType, ipFamilies []v1.IPFamily) *v1.Service { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: ns, }, Spec: v1.ServiceSpec{ - Selector: labels, - Type: v1.ServiceTypeNodePort, - IPFamily: ipFamily, + Selector: labels, + Type: v1.ServiceTypeNodePort, + IPFamilyPolicy: ipFamilyPolicy, + IPFamilies: ipFamilies, Ports: []v1.ServicePort{ { Name: "tcp-port", diff --git a/test/integration/BUILD b/test/integration/BUILD index baeeece5b5c..901d12ef8dc 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -50,6 +50,7 @@ filegroup( "//test/integration/deployment:all-srcs", "//test/integration/disruption:all-srcs", "//test/integration/dryrun:all-srcs", + "//test/integration/dualstack:all-srcs", "//test/integration/endpoints:all-srcs", "//test/integration/endpointslice:all-srcs", "//test/integration/etcd:all-srcs", diff --git a/test/integration/dualstack/BUILD b/test/integration/dualstack/BUILD new file mode 100644 index 00000000000..78da5d72c04 --- /dev/null +++ b/test/integration/dualstack/BUILD @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "go_default_test", + srcs = [ + "dualstack_test.go", + "main_test.go", + ], + tags = ["integration"], + deps = [ + "//pkg/features:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//test/integration/framework:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/integration/dualstack/dualstack_test.go b/test/integration/dualstack/dualstack_test.go new file mode 100644 index 00000000000..3cb447571af --- /dev/null +++ b/test/integration/dualstack/dualstack_test.go @@ -0,0 +1,1353 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dualstack + +import ( + "context" + "fmt" + "net" + "reflect" + "strings" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/integration/framework" + netutils "k8s.io/utils/net" +) + +// TestCreateServiceSingleStackIPv4 test the Service dualstackness in an IPv4 SingleStack cluster +func TestCreateServiceSingleStackIPv4(t *testing.T) { + // Create an IPv4 single stack control-plane + serviceCIDR := "10.0.0.0/16" + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicyType + expectedIPFamilies []v1.IPFamily + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: nil, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: nil, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectError: true, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + + // create the service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s\n%+v\n%v", svc.Name, svc, err) + } + }) + } +} + +// TestCreateServiceDualStackIPv6 test the Service dualstackness in an IPv6 only DualStack cluster +func TestCreateServiceDualStackIPv6(t *testing.T) { + // Create an IPv6 only dual stack control-plane + serviceCIDR := "2001:db8:1::/48" + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicyType + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: true, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Name: fmt.Sprintf("port-test-%d", i), + Port: 443, + TargetPort: intstr.IntOrString{IntVal: 443}, + Protocol: "TCP", + }, + }, + }, + } + + // create the service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } +} + +// TestCreateServiceDualStackIPv4IPv6 test the Service dualstackness in a IPv4IPv6 DualStack cluster +func TestCreateServiceDualStackIPv4IPv6(t *testing.T) { + // Create an IPv4IPv6 dual stack control-plane + serviceCIDR := "10.0.0.0/16" + secondaryServiceCIDR := "2001:db8:1::/48" + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + + _, secCidr, err := net.ParseCIDR(secondaryServiceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.SecondaryServiceIPRange = *secCidr + + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicyType + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + + // create a service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } +} + +// TestCreateServiceDualStackIPv6IPv4 test the Service dualstackness in a IPv6IPv4 DualStack cluster +func TestCreateServiceDualStackIPv6IPv4(t *testing.T) { + // Create an IPv6IPv4 dual stack control-plane + serviceCIDR := "2001:db8:1::/48" + secondaryServiceCIDR := "10.0.0.0/16" + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + + _, secCidr, err := net.ParseCIDR(secondaryServiceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.SecondaryServiceIPRange = *secCidr + + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + // verify client is working + if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { + _, err = client.CoreV1().Endpoints("default").Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil { + t.Logf("error fetching endpoints: %v", err) + return false, nil + } + return true, nil + }); err != nil { + t.Errorf("server without enabled endpoints failed to register: %v", err) + } + + var testcases = []struct { + name string + serviceType v1.ServiceType + clusterIPs []string + ipFamilies []v1.IPFamily + expectedIPFamilies []v1.IPFamily + ipFamilyPolicy v1.IPFamilyPolicyType + expectError bool + }{ + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: nil, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicySingleStack, + expectError: true, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyPreferDualStack, + expectError: false, + }, + { + name: "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack", + serviceType: v1.ServiceTypeClusterIP, + clusterIPs: []string{}, + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, + expectError: false, + }, + } + + for i, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test + }, + Spec: v1.ServiceSpec{ + Type: tc.serviceType, + ClusterIPs: tc.clusterIPs, + IPFamilies: tc.ipFamilies, + IPFamilyPolicy: &tc.ipFamilyPolicy, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + + // create a service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if (err != nil) != tc.expectError { + t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err) + } + // if no error was expected validate the service otherwise return + if err != nil { + return + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil { + t.Errorf("Unexpected error validating the service %s %v", svc.Name, err) + } + }) + } +} + +// TestUpgradeDowngrade tests upgrading and downgrading a service from/to dual-stack +func TestUpgradeDowngrade(t *testing.T) { + // Create an IPv4IPv6 dual stack control-plane + serviceCIDR := "10.0.0.0/16" + secondaryServiceCIDR := "2001:db8:1::/48" + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + + _, secCidr, err := net.ParseCIDR(secondaryServiceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.SecondaryServiceIPRange = *secCidr + + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + upgradeServiceName := "svc-upgrade" + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: upgradeServiceName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + + // create a service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("unexpected error while creating service:%v", err) + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol} /* default cluster config */); err != nil { + t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err) + } + + // upgrade it + requireDualStack := v1.IPFamilyPolicyRequireDualStack + svc.Spec.IPFamilyPolicy = &requireDualStack + upgraded, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(context.TODO(), svc, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unexpected error upgrading service to dual stack. %v", err) + } + if err := validateServiceAndClusterIPFamily(upgraded, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} /* +1 family */); err != nil { + t.Fatalf("Unexpected error validating the service(after upgrade) %s %v", svc.Name, err) + } + + // downgrade it + singleStack := v1.IPFamilyPolicySingleStack + upgraded.Spec.IPFamilyPolicy = &singleStack + upgraded.Spec.ClusterIPs = upgraded.Spec.ClusterIPs[0:1] + upgraded.Spec.IPFamilies = upgraded.Spec.IPFamilies[0:1] + downgraded, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(context.TODO(), upgraded, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unexpected error downgrading service to single stack. %v", err) + } + if err := validateServiceAndClusterIPFamily(downgraded, []v1.IPFamily{v1.IPv4Protocol} /* -1 family */); err != nil { + t.Fatalf("unexpected error validating the service(after downgrade) %s %v", svc.Name, err) + } + + // run test again this time without removing secondary IPFamily or ClusterIP + downgraded.Spec.IPFamilyPolicy = &requireDualStack + upgradedAgain, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(context.TODO(), downgraded, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unexpected error upgrading service to dual stack. %v", err) + } + if err := validateServiceAndClusterIPFamily(upgradedAgain, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} /* +1 family */); err != nil { + t.Fatalf("Unexpected error validating the service(after upgrade) %s %v", svc.Name, err) + } + + upgradedAgain.Spec.IPFamilyPolicy = &singleStack + // api-server automatically removes the secondary ClusterIP and IPFamily + // when a servie is downgraded. + downgradedAgain, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(context.TODO(), upgradedAgain, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unexpected error downgrading service to single stack. %v", err) + } + if err := validateServiceAndClusterIPFamily(downgradedAgain, []v1.IPFamily{v1.IPv4Protocol} /* -1 family */); err != nil { + t.Fatalf("unexpected error validating the service(after downgrade) %s %v", svc.Name, err) + } +} + +// TestConvertToFromExternalName tests the compatibility with old clients that +// may not clear ClusterIPs +func TestConvertToFromExternalName(t *testing.T) { + // Create an IPv4IPv6 dual stack control-plane + serviceCIDR := "10.0.0.0/16" + secondaryServiceCIDR := "2001:db8:1::/48" + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + + _, secCidr, err := net.ParseCIDR(secondaryServiceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.SecondaryServiceIPRange = *secCidr + + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + serviceName := "svc-ext-name" + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + + // create a service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("unexpected error while creating service:%v", err) + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol}); err != nil { + t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err) + } + + // convert to ExternalName + svc.Spec.Type = v1.ServiceTypeExternalName + svc.Spec.ClusterIP = "" // not clearing ClusterIPs + svc.Spec.ExternalName = "something.somewhere" + + externalNameSvc, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(context.TODO(), svc, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unexpected error converting service to external name. %v", err) + } + + if len(externalNameSvc.Spec.ClusterIPs) > 0 || len(externalNameSvc.Spec.ClusterIP) > 0 || len(externalNameSvc.Spec.IPFamilies) > 0 { + t.Fatalf("unpexpected externalname service with ClusterIPs %v or ClusterIP %v or IPFamilies %v", externalNameSvc.Spec.ClusterIPs, externalNameSvc.Spec.ClusterIP, externalNameSvc.Spec.IPFamilies) + } + + // convert to a ClusterIP service + externalNameSvc.Spec.Type = v1.ServiceTypeClusterIP + externalNameSvc.Spec.ExternalName = "" + clusterIPSvc, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(context.TODO(), externalNameSvc, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unexpected error converting service to ClusterIP. %v", err) + } + if err := validateServiceAndClusterIPFamily(clusterIPSvc, []v1.IPFamily{v1.IPv4Protocol}); err != nil { + t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err) + } +} + +// TestExistingServiceDefaulting tests that existing services are defaulted correctly after an upgrade +func TestExistingServiceDefaulting(t *testing.T) { + // Create an IPv4IPv6 dual stack control-plane + serviceCIDR := "10.0.0.0/16" + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + serviceName := "svc-ext-name" + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + // make sure gate is off (cluster is not running as dual stack) + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, false)() + + // create a service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("unexpected error while creating service:%v", err) + } + + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err) + } + + // turn gate on + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + // validate the service was created correctly if it was not expected to fail + defaultedSvc, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if defaultedSvc.Spec.IPFamilyPolicy == nil { + t.Fatalf("service should have an IPFamily Policy") + } else { + // must be equal to single stack + if *(defaultedSvc.Spec.IPFamilyPolicy) != v1.IPFamilyPolicySingleStack { + t.Fatalf("service should have a SingleStack as IPFamilyPolicy instead we got:%v", defaultedSvc.Spec.IPFamilyPolicy) + } + } + + if err := validateServiceAndClusterIPFamily(defaultedSvc, []v1.IPFamily{v1.IPv4Protocol}); err != nil { + t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err) + } +} + +// TestPreferDualStack preferDualstack on create and update +func TestPreferDualStack(t *testing.T) { + // Create an IPv4IPv6 dual stack control-plane + serviceCIDR := "10.0.0.0/16" + secondaryServiceCIDR := "2001:db8:1::/48" + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, true)() + + cfg := framework.NewIntegrationTestMasterConfig() + _, cidr, err := net.ParseCIDR(serviceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.ServiceIPRange = *cidr + + _, secCidr, err := net.ParseCIDR(secondaryServiceCIDR) + if err != nil { + t.Fatalf("bad cidr: %v", err) + } + cfg.ExtraConfig.SecondaryServiceIPRange = *secCidr + + _, s, closeFn := framework.RunAMaster(cfg) + defer closeFn() + + client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + + // Wait until the default "kubernetes" service is created. + if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + return !apierrors.IsNotFound(err), nil + }); err != nil { + t.Fatalf("creating kubernetes service timed out") + } + + preferDualStack := v1.IPFamilyPolicyPreferDualStack + + serviceName := "svc-upgrade" + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + IPFamilyPolicy: &preferDualStack, + Ports: []v1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + + // create a service + _, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), svc, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("unexpected error while creating service:%v", err) + } + // validate the service was created correctly if it was not expected to fail + svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err) + } + + if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil { + t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err) + } + + // update it + svc.Spec.Selector = map[string]string{"foo": "bar"} + upgraded, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(context.TODO(), svc, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("unexpected error upgrading service to dual stack. %v", err) + } + if err := validateServiceAndClusterIPFamily(upgraded, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil { + t.Fatalf("Unexpected error validating the service(after upgrade) %s %v", svc.Name, err) + } +} + +// validateServiceAndClusterIPFamily checks that the service has the expected IPFamilies +func validateServiceAndClusterIPFamily(svc *v1.Service, expectedIPFamilies []v1.IPFamily) error { + // create a slice for the errors + var errstrings []string + + if svc.Spec.IPFamilies == nil { + return fmt.Errorf("service ip family nil for service %s/%s", svc.Namespace, svc.Name) + } + if !reflect.DeepEqual(svc.Spec.IPFamilies, expectedIPFamilies) { + return fmt.Errorf("ip families mismatch for service: %s/%s, expected: %s, actual: %s", svc.Namespace, svc.Name, expectedIPFamilies, svc.Spec.IPFamilies) + } + + if len(svc.Spec.ClusterIPs) == 0 { + return fmt.Errorf("svc %s is invalid it does not have ClusterIP", svc.Name) + } + + // not headless + if len(svc.Spec.ClusterIPs) > 0 && svc.Spec.ClusterIPs[0] != v1.ClusterIPNone { + if len(svc.Spec.ClusterIPs) != len(svc.Spec.IPFamilies) { + return fmt.Errorf("svc %v is invalid len(ClusterIPs:%v) != len(IPFamilies:%v)", svc.Name, svc.Spec.ClusterIPs, svc.Spec.IPFamilies) + } + } + + for j, ip := range svc.Spec.ClusterIPs { + // we should never be here + if ip == v1.ClusterIPNone && len(svc.Spec.ClusterIPs) > 1 { + errstrings = append(errstrings, fmt.Sprintf("Error validating Service: %s, None is used with +1 clusterIPs (%v)", svc.Name, svc.Spec.ClusterIPs)) + } + + if ip == v1.ClusterIPNone { + break // the service is headless. the rest of family check is pointless + } + + // the clusterIP assigned should have the same IPFamily requested + if netutils.IsIPv6String(ip) != (expectedIPFamilies[j] == v1.IPv6Protocol) { + errstrings = append(errstrings, fmt.Sprintf("got unexpected service ip %s, should belong to %s ip family", ip, expectedIPFamilies[j])) + } + } + + if len(errstrings) > 0 { + errstrings = append(errstrings, fmt.Sprintf("Error validating Service: %s, ClusterIPs: %v Expected IPFamilies %v", svc.Name, svc.Spec.ClusterIPs, expectedIPFamilies)) + return fmt.Errorf(strings.Join(errstrings, "\n")) + } + + return nil +} diff --git a/test/integration/dualstack/main_test.go b/test/integration/dualstack/main_test.go new file mode 100644 index 00000000000..6f6c876f2a3 --- /dev/null +++ b/test/integration/dualstack/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dualstack + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +}