diff --git a/pkg/api/v1/service/util.go b/pkg/api/v1/service/util.go index 68913287c13..928c82c34f6 100644 --- a/pkg/api/v1/service/util.go +++ b/pkg/api/v1/service/util.go @@ -67,8 +67,8 @@ func GetLoadBalancerSourceRanges(service *v1.Service) (utilnet.IPNetSet, error) return ipnets, nil } -// RequestsOnlyLocalTraffic checks if service requests OnlyLocal traffic. -func RequestsOnlyLocalTraffic(service *v1.Service) bool { +// ExternalPolicyLocal checks if service has ETP = Local. +func ExternalPolicyLocal(service *v1.Service) bool { if service.Spec.Type != v1.ServiceTypeLoadBalancer && service.Spec.Type != v1.ServiceTypeNodePort { return false @@ -76,9 +76,8 @@ func RequestsOnlyLocalTraffic(service *v1.Service) bool { return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal } -// RequestsOnlyLocalTrafficForInternal checks if service prefers Node Local -// endpoints for internal traffic -func RequestsOnlyLocalTrafficForInternal(service *v1.Service) bool { +// InternalPolicyLocal checks if service has ITP = Local. +func InternalPolicyLocal(service *v1.Service) bool { if service.Spec.InternalTrafficPolicy == nil { return false } @@ -90,7 +89,7 @@ func NeedsHealthCheck(service *v1.Service) bool { if service.Spec.Type != v1.ServiceTypeLoadBalancer { return false } - return RequestsOnlyLocalTraffic(service) + return ExternalPolicyLocal(service) } // GetServiceHealthCheckPathPort returns the path and nodePort programmed into the Cloud LB Health Check diff --git a/pkg/api/v1/service/util_test.go b/pkg/api/v1/service/util_test.go index f7a3c2dadb9..42510c28897 100644 --- a/pkg/api/v1/service/util_test.go +++ b/pkg/api/v1/service/util_test.go @@ -129,45 +129,45 @@ func TestAllowAll(t *testing.T) { checkAllowAll(true, "192.168.0.1/32", "0.0.0.0/0") } -func TestRequestsOnlyLocalTraffic(t *testing.T) { - checkRequestsOnlyLocalTraffic := func(requestsOnlyLocalTraffic bool, service *v1.Service) { - res := RequestsOnlyLocalTraffic(service) +func TestExternalPolicyLocal(t *testing.T) { + checkExternalPolicyLocal := func(requestsOnlyLocalTraffic bool, service *v1.Service) { + res := ExternalPolicyLocal(service) if res != requestsOnlyLocalTraffic { t.Errorf("Expected requests OnlyLocal traffic = %v, got %v", requestsOnlyLocalTraffic, res) } } - checkRequestsOnlyLocalTraffic(false, &v1.Service{}) - checkRequestsOnlyLocalTraffic(false, &v1.Service{ + checkExternalPolicyLocal(false, &v1.Service{}) + checkExternalPolicyLocal(false, &v1.Service{ Spec: v1.ServiceSpec{ Type: v1.ServiceTypeClusterIP, }, }) - checkRequestsOnlyLocalTraffic(false, &v1.Service{ + checkExternalPolicyLocal(false, &v1.Service{ Spec: v1.ServiceSpec{ Type: v1.ServiceTypeNodePort, }, }) - checkRequestsOnlyLocalTraffic(false, &v1.Service{ + checkExternalPolicyLocal(false, &v1.Service{ Spec: v1.ServiceSpec{ Type: v1.ServiceTypeNodePort, ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster, }, }) - checkRequestsOnlyLocalTraffic(true, &v1.Service{ + checkExternalPolicyLocal(true, &v1.Service{ Spec: v1.ServiceSpec{ Type: v1.ServiceTypeNodePort, ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, }, }) - checkRequestsOnlyLocalTraffic(false, &v1.Service{ + checkExternalPolicyLocal(false, &v1.Service{ Spec: v1.ServiceSpec{ Type: v1.ServiceTypeLoadBalancer, ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster, }, }) - checkRequestsOnlyLocalTraffic(true, &v1.Service{ + checkExternalPolicyLocal(true, &v1.Service{ Spec: v1.ServiceSpec{ Type: v1.ServiceTypeLoadBalancer, ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, @@ -215,9 +215,9 @@ func TestNeedsHealthCheck(t *testing.T) { }) } -func TestRequestsOnlyLocalTrafficForInternal(t *testing.T) { - checkRequestsOnlyLocalTrafficForInternal := func(expected bool, service *v1.Service) { - res := RequestsOnlyLocalTrafficForInternal(service) +func TestInternalPolicyLocal(t *testing.T) { + checkInternalPolicyLocal := func(expected bool, service *v1.Service) { + res := InternalPolicyLocal(service) if res != expected { t.Errorf("Expected internal local traffic = %v, got %v", expected, res) @@ -225,17 +225,17 @@ func TestRequestsOnlyLocalTrafficForInternal(t *testing.T) { } // default InternalTrafficPolicy is nil - checkRequestsOnlyLocalTrafficForInternal(false, &v1.Service{}) + checkInternalPolicyLocal(false, &v1.Service{}) local := v1.ServiceInternalTrafficPolicyLocal - checkRequestsOnlyLocalTrafficForInternal(true, &v1.Service{ + checkInternalPolicyLocal(true, &v1.Service{ Spec: v1.ServiceSpec{ InternalTrafficPolicy: &local, }, }) cluster := v1.ServiceInternalTrafficPolicyCluster - checkRequestsOnlyLocalTrafficForInternal(false, &v1.Service{ + checkInternalPolicyLocal(false, &v1.Service{ Spec: v1.ServiceSpec{ InternalTrafficPolicy: &cluster, }, diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 3d372a779a9..8692e3ae7c2 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -115,11 +115,11 @@ const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables" type serviceInfo struct { *proxy.BaseServiceInfo // The following fields are computed and stored for performance reasons. - serviceNameString string - servicePortChainName utiliptables.Chain - serviceLocalChainName utiliptables.Chain - serviceFirewallChainName utiliptables.Chain - serviceLBChainName utiliptables.Chain + nameString string + clusterPolicyChainName utiliptables.Chain + localPolicyChainName utiliptables.Chain + firewallChainName utiliptables.Chain + externalChainName utiliptables.Chain } // returns a new proxy.ServicePort which abstracts a serviceInfo @@ -130,11 +130,11 @@ func newServiceInfo(port *v1.ServicePort, service *v1.Service, baseInfo *proxy.B svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name} protocol := strings.ToLower(string(info.Protocol())) - info.serviceNameString = svcPortName.String() - info.servicePortChainName = servicePortChainName(info.serviceNameString, protocol) - info.serviceLocalChainName = serviceLocalChainName(info.serviceNameString, protocol) - info.serviceFirewallChainName = serviceFirewallChainName(info.serviceNameString, protocol) - info.serviceLBChainName = serviceLBChainName(info.serviceNameString, protocol) + info.nameString = svcPortName.String() + info.clusterPolicyChainName = servicePortPolicyClusterChain(info.nameString, protocol) + info.localPolicyChainName = servicePortPolicyLocalChainName(info.nameString, protocol) + info.firewallChainName = serviceFirewallChainName(info.nameString, protocol) + info.externalChainName = serviceExternalChainName(info.nameString, protocol) return info } @@ -684,25 +684,28 @@ func portProtoHash(servicePortName string, protocol string) string { } const ( - servicePortChainNamePrefix = "KUBE-SVC-" - serviceLocalChainNamePrefix = "KUBE-SVL-" - serviceFirewallChainNamePrefix = "KUBE-FW-" - serviceLBChainNamePrefix = "KUBE-XLB-" - servicePortEndpointChainNamePrefix = "KUBE-SEP-" + servicePortPolicyClusterChainNamePrefix = "KUBE-SVC-" + servicePortPolicyLocalChainNamePrefix = "KUBE-SVL-" + serviceFirewallChainNamePrefix = "KUBE-FW-" + serviceExternalChainNamePrefix = "KUBE-EXT-" + servicePortEndpointChainNamePrefix = "KUBE-SEP-" + + // For cleanup. This can be removed after 1.26 is released. + deprecatedServiceLBChainNamePrefix = "KUBE-XLB-" ) -// servicePortChainName returns the name of the KUBE-SVC-XXXX chain for a service, which is the +// servicePortPolicyClusterChain returns the name of the KUBE-SVC-XXXX chain for a service, which is the // main iptables chain for that service, used for dispatching to endpoints when using `Cluster` // traffic policy. -func servicePortChainName(servicePortName string, protocol string) utiliptables.Chain { - return utiliptables.Chain(servicePortChainNamePrefix + portProtoHash(servicePortName, protocol)) +func servicePortPolicyClusterChain(servicePortName string, protocol string) utiliptables.Chain { + return utiliptables.Chain(servicePortPolicyClusterChainNamePrefix + portProtoHash(servicePortName, protocol)) } -// serviceLocalChainName returns the name of the KUBE-SVL-XXXX chain for a service, which +// servicePortPolicyLocalChainName returns the name of the KUBE-SVL-XXXX chain for a service, which // handles dispatching to local endpoints when using `Local` traffic policy. This chain only // exists if the service has `Local` internal or external traffic policy. -func serviceLocalChainName(servicePortName string, protocol string) utiliptables.Chain { - return utiliptables.Chain(serviceLocalChainNamePrefix + portProtoHash(servicePortName, protocol)) +func servicePortPolicyLocalChainName(servicePortName string, protocol string) utiliptables.Chain { + return utiliptables.Chain(servicePortPolicyLocalChainNamePrefix + portProtoHash(servicePortName, protocol)) } // serviceFirewallChainName returns the name of the KUBE-FW-XXXX chain for a service, which @@ -711,12 +714,12 @@ func serviceFirewallChainName(servicePortName string, protocol string) utiliptab return utiliptables.Chain(serviceFirewallChainNamePrefix + portProtoHash(servicePortName, protocol)) } -// serviceLBChainName returns the name of the KUBE-XLB-XXXX chain for a service, which -// implements "short-circuiting" for internally-originated load balancer traffic when using -// `Local` external traffic policy. It forwards traffic from local sources to the KUBE-SVC-XXXX +// serviceExternalChainName returns the name of the KUBE-EXT-XXXX chain for a service, which +// implements "short-circuiting" for internally-originated external-destination traffic when using +// `Local` external traffic policy. It forwards traffic from local sources to the KUBE-SVC-XXXX // chain and traffic from external sources to the KUBE-SVL-XXXX chain. -func serviceLBChainName(servicePortName string, protocol string) utiliptables.Chain { - return utiliptables.Chain(serviceLBChainNamePrefix + portProtoHash(servicePortName, protocol)) +func serviceExternalChainName(servicePortName string, protocol string) utiliptables.Chain { + return utiliptables.Chain(serviceExternalChainNamePrefix + portProtoHash(servicePortName, protocol)) } // servicePortEndpointChainName returns the name of the KUBE-SEP-XXXX chain for a particular @@ -729,11 +732,12 @@ func servicePortEndpointChainName(servicePortName string, protocol string, endpo func isServiceChainName(chainString string) bool { prefixes := []string{ - servicePortChainNamePrefix, - serviceLocalChainNamePrefix, + servicePortPolicyClusterChainNamePrefix, + servicePortPolicyLocalChainNamePrefix, servicePortEndpointChainNamePrefix, serviceFirewallChainNamePrefix, - serviceLBChainNamePrefix, + serviceExternalChainNamePrefix, + deprecatedServiceLBChainNamePrefix, } for _, p := range prefixes { @@ -989,16 +993,15 @@ func (proxier *Proxier) syncProxyRules() { } } - // Build rules for each service. + // Build rules for each service-port. for svcName, svc := range proxier.serviceMap { svcInfo, ok := svc.(*serviceInfo) if !ok { klog.ErrorS(nil, "Failed to cast serviceInfo", "serviceName", svcName) continue } - isIPv6 := netutils.IsIPv6(svcInfo.ClusterIP()) protocol := strings.ToLower(string(svcInfo.Protocol())) - svcNameString := svcInfo.serviceNameString + svcNameString := svcInfo.nameString allEndpoints := proxier.endpointsMap[svcName] @@ -1042,80 +1045,108 @@ func (proxier *Proxier) syncProxyRules() { proxier.natRules.Write(args) } - policyClusterChain := svcInfo.servicePortChainName - policyLocalChain := svcInfo.serviceLocalChainName - svcXlbChain := svcInfo.serviceLBChainName + // These chains represent the sets of endpoints to use when internal or + // external traffic policy is "Cluster" vs "Local". + clusterPolicyChain := svcInfo.clusterPolicyChainName + localPolicyChain := svcInfo.localPolicyChainName - internalTrafficChain := policyClusterChain - externalTrafficChain := policyClusterChain - - if svcInfo.NodeLocalInternal() { - internalTrafficChain = policyLocalChain + // These chains designate which policy chain to use for internal- and + // external-destination traffic. + internalPolicyChain := clusterPolicyChain + externalPolicyChain := clusterPolicyChain + if svcInfo.InternalPolicyLocal() { + internalPolicyChain = localPolicyChain } - if svcInfo.NodeLocalExternal() { - externalTrafficChain = svcXlbChain + if svcInfo.ExternalPolicyLocal() { + externalPolicyChain = localPolicyChain } + // These chains are where *ALL* rules which match traffic that is + // service-destined should jump. ClusterIP traffic is considered + // "internal" while NodePort, LoadBalancer, and ExternalIPs traffic is + // considered "external". + internalTrafficChain := internalPolicyChain + externalTrafficChain := svcInfo.externalChainName // eventually jumps to externalPolicyChain + + // Declare the clusterPolicyChain if needed. if hasEndpoints && svcInfo.UsesClusterEndpoints() { // Create the Cluster traffic policy chain, retaining counters if possible. - if chain, ok := existingNATChains[policyClusterChain]; ok { + if chain, ok := existingNATChains[clusterPolicyChain]; ok { proxier.natChains.WriteBytes(chain) } else { - proxier.natChains.Write(utiliptables.MakeChainLine(policyClusterChain)) + proxier.natChains.Write(utiliptables.MakeChainLine(clusterPolicyChain)) } - activeNATChains[policyClusterChain] = true - } - - if hasEndpoints && svcInfo.ExternallyAccessible() && svcInfo.NodeLocalExternal() { - if chain, ok := existingNATChains[svcXlbChain]; ok { - proxier.natChains.WriteBytes(chain) - } else { - proxier.natChains.Write(utiliptables.MakeChainLine(svcXlbChain)) - } - activeNATChains[svcXlbChain] = true - - // The XLB chain redirects all pod -> external VIP - // traffic to the Service's ClusterIP instead. This happens - // whether or not we have local endpoints; only if localDetector - // is implemented - if proxier.localDetector.IsImplemented() { - proxier.natRules.Write( - "-A", string(svcXlbChain), - "-m", "comment", "--comment", - `"Redirect pods trying to reach external loadbalancer VIP to clusterIP"`, - proxier.localDetector.IfLocal(), - "-j", string(policyClusterChain)) - } - - // Next, redirect all src-type=LOCAL -> LB IP to the service chain - // for externalTrafficPolicy=Local This allows traffic originating - // from the host to be redirected to the service correctly, - // otherwise traffic to LB IPs are dropped if there are no local - // endpoints. - proxier.natRules.Write( - "-A", string(svcXlbChain), - "-m", "comment", "--comment", fmt.Sprintf(`"masquerade LOCAL traffic for %s LB IP"`, svcNameString), - "-m", "addrtype", "--src-type", "LOCAL", - "-j", string(KubeMarkMasqChain)) - proxier.natRules.Write( - "-A", string(svcXlbChain), - "-m", "comment", "--comment", fmt.Sprintf(`"route LOCAL traffic for %s LB IP to service chain"`, svcNameString), - "-m", "addrtype", "--src-type", "LOCAL", - "-j", string(policyClusterChain)) - - // Everything else goes to the SVL chain - proxier.natRules.Write( - "-A", string(svcXlbChain), - "-j", string(policyLocalChain)) + activeNATChains[clusterPolicyChain] = true } + // Declare the localPolicyChain if needed. if hasEndpoints && svcInfo.UsesLocalEndpoints() { - if chain, ok := existingNATChains[policyLocalChain]; ok { + if chain, ok := existingNATChains[localPolicyChain]; ok { proxier.natChains.WriteBytes(chain) } else { - proxier.natChains.Write(utiliptables.MakeChainLine(policyLocalChain)) + proxier.natChains.Write(utiliptables.MakeChainLine(localPolicyChain)) } - activeNATChains[policyLocalChain] = true + activeNATChains[localPolicyChain] = true + } + + // If any "external" destinations are enabled, set up external traffic + // handling. All captured traffic for all external destinations should + // jump to externalTrafficChain, which will handle some special-cases + // and then jump to externalPolicyChain. + if hasEndpoints && svcInfo.ExternallyAccessible() { + if chain, ok := existingNATChains[externalTrafficChain]; ok { + proxier.natChains.WriteBytes(chain) + } else { + proxier.natChains.Write(utiliptables.MakeChainLine(externalTrafficChain)) + } + activeNATChains[externalTrafficChain] = true + + if !svcInfo.ExternalPolicyLocal() { + // If we are using non-local endpoints we need to masquerade, + // in case we cross nodes. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"masquerade traffic for %s external destinations"`, svcNameString), + "-j", string(KubeMarkMasqChain)) + } else { + // If we are only using same-node endpoints, we can retain the + // source IP in most cases. + + if proxier.localDetector.IsImplemented() { + // Treat all locally-originated pod -> external destination + // traffic as a special-case. It is subject to neither + // form of traffic policy, which simulates going up-and-out + // to an external load-balancer and coming back in. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"pod traffic for %s external destinations"`, svcNameString), + proxier.localDetector.IfLocal(), + "-j", string(clusterPolicyChain)) + } + + // Locally originated traffic (not a pod, but the host node) + // still needs masquerade because the LBIP itself is a local + // address, so that will be the chosen source IP. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"masquerade LOCAL traffic for %s external destinations"`, svcNameString), + "-m", "addrtype", "--src-type", "LOCAL", + "-j", string(KubeMarkMasqChain)) + + // Redirect all src-type=LOCAL -> external destination to the + // policy=cluster chain. This allows traffic originating + // from the host to be redirected to the service correctly. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-m", "comment", "--comment", fmt.Sprintf(`"route LOCAL traffic for %s external destinations"`, svcNameString), + "-m", "addrtype", "--src-type", "LOCAL", + "-j", string(clusterPolicyChain)) + } + + // Anything else falls thru to the appropriate policy chain. + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-j", string(externalPolicyChain)) } // Capture the clusterIP. @@ -1161,37 +1192,14 @@ func (proxier *Proxier) syncProxyRules() { // Capture externalIPs. for _, externalIP := range svcInfo.ExternalIPStrings() { if hasEndpoints { - args = append(args[:0], + // Send traffic bound for external IPs to the "external + // destinations" chain. + proxier.natRules.Write( + "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s external IP"`, svcNameString), "-m", protocol, "-p", protocol, "-d", externalIP, "--dport", strconv.Itoa(svcInfo.Port()), - ) - - // We have to SNAT packets to external IPs if externalTrafficPolicy is cluster - // and the traffic is NOT Local. Local traffic coming from Pods and Nodes will - // be always forwarded to the corresponding Service, so no need to SNAT - // If we can't differentiate the local traffic we always SNAT. - if !svcInfo.NodeLocalExternal() { - appendTo := []string{"-A", string(policyClusterChain)} - // This masquerades off-cluster traffic to a External IP. - if proxier.localDetector.IsImplemented() { - proxier.natRules.Write( - appendTo, - args, - proxier.localDetector.IfNotLocal(), - "-j", string(KubeMarkMasqChain)) - } else { - proxier.natRules.Write( - appendTo, - args, - "-j", string(KubeMarkMasqChain)) - } - } - // Send traffic bound for external IPs to the service chain. - proxier.natRules.Write( - "-A", string(kubeServicesChain), - args, "-j", string(externalTrafficChain)) } else { @@ -1208,45 +1216,48 @@ func (proxier *Proxier) syncProxyRules() { } // Capture load-balancer ingress. - fwChain := svcInfo.serviceFirewallChainName - for _, ingress := range svcInfo.LoadBalancerIPStrings() { - if hasEndpoints { - // create service firewall chain + if len(svcInfo.LoadBalancerIPStrings()) > 0 && hasEndpoints { + // Normally we send LB matches to the "external destination" chain. + nextChain := externalTrafficChain + + // If the service specifies any LB source ranges, we need to insert + // a firewall chain first. + if len(svcInfo.LoadBalancerSourceRanges()) > 0 { + fwChain := svcInfo.firewallChainName + + // Declare the service firewall chain. if chain, ok := existingNATChains[fwChain]; ok { proxier.natChains.WriteBytes(chain) } else { proxier.natChains.Write(utiliptables.MakeChainLine(fwChain)) } activeNATChains[fwChain] = true - // The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field. - // This currently works for loadbalancers that preserves source ips. - // For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply. - args = append(args[:0], + // The firewall chain will jump to the "external destination" + // chain. + nextChain = svcInfo.firewallChainName + } + + for _, lbip := range svcInfo.LoadBalancerIPStrings() { + proxier.natRules.Write( "-A", string(kubeServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString), "-m", protocol, "-p", protocol, - "-d", ingress, + "-d", lbip, "--dport", strconv.Itoa(svcInfo.Port()), - ) - // jump to service firewall chain - proxier.natRules.Write(args, "-j", string(fwChain)) + "-j", string(nextChain)) - args = append(args[:0], - "-A", string(fwChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString), - ) + // The service firewall rules are created based on the + // loadBalancerSourceRanges field. This only works for + // VIP-like loadbalancers that preserve source IPs. For + // loadbalancers which direct traffic to service NodePort, the + // firewall rules will not apply. + if len(svcInfo.LoadBalancerSourceRanges()) > 0 { + args = append(args[:0], + "-A", string(nextChain), + "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString), + ) - // If we are proxying globally, we need to masquerade in case we cross nodes. - // If we are proxying only locally, we can retain the source IP. - if !svcInfo.NodeLocalExternal() { - proxier.natRules.Write(args, "-j", string(KubeMarkMasqChain)) - } - - if len(svcInfo.LoadBalancerSourceRanges()) == 0 { - // allow all sources, so jump directly to the KUBE-SVC or KUBE-XLB chain - proxier.natRules.Write(args, "-j", string(externalTrafficChain)) - } else { // firewall filter based on each source range allowFromNode := false for _, src := range svcInfo.LoadBalancerSourceRanges() { @@ -1258,67 +1269,48 @@ func (proxier *Proxier) syncProxyRules() { allowFromNode = true } } - // generally, ip route rule was added to intercept request to loadbalancer vip from the - // loadbalancer's backend hosts. In this case, request will not hit the loadbalancer but loop back directly. - // Need to add the following rule to allow request on host. + // For VIP-like LBs, the VIP is often added as a local + // address (via an IP route rule). In that case, a request + // from a node to the VIP will not hit the loadbalancer but + // will loop back with the source IP set to the VIP. We + // need the following rule to allow requests from this node. if allowFromNode { proxier.natRules.Write( args, - "-s", ingress, + "-s", lbip, "-j", string(externalTrafficChain)) } + // If the packet was able to reach the end of firewall chain, + // then it did not get DNATed. It means the packet cannot go + // thru the firewall, then mark it for DROP. + proxier.natRules.Write(args, "-j", string(KubeMarkDropChain)) } - - // If the packet was able to reach the end of firewall chain, then it did not get DNATed. - // It means the packet cannot go thru the firewall, then mark it for DROP - proxier.natRules.Write(args, "-j", string(KubeMarkDropChain)) - } else { - // No endpoints. + } + } else { + // No endpoints. + for _, lbip := range svcInfo.LoadBalancerIPStrings() { proxier.filterRules.Write( "-A", string(kubeExternalServicesChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString), "-m", protocol, "-p", protocol, - "-d", ingress, + "-d", lbip, "--dport", strconv.Itoa(svcInfo.Port()), "-j", "REJECT", ) } } - // Capture nodeports. If we had more than 2 rules it might be - // worthwhile to make a new per-service chain for nodeport rules, but - // with just 2 rules it ends up being a waste and a cognitive burden. + // Capture nodeports. if svcInfo.NodePort() != 0 && len(nodeAddresses) != 0 { if hasEndpoints { - args = append(args[:0], + // Jump to the external destination chain. For better or for + // worse, nodeports are not subect to loadBalancerSourceRanges, + // and we can't change that. + proxier.natRules.Write( + "-A", string(kubeNodePortsChain), "-m", "comment", "--comment", svcNameString, "-m", protocol, "-p", protocol, "--dport", strconv.Itoa(svcInfo.NodePort()), - ) - if !svcInfo.NodeLocalExternal() { - // Nodeports need SNAT, unless they're local. - proxier.natRules.Write( - "-A", string(policyClusterChain), - args, - "-j", string(KubeMarkMasqChain)) - } else { - // TODO: Make all nodePorts jump to the firewall chain. - // Currently we only create it for loadbalancers (#33586). - - // Fix localhost martian source error - loopback := "127.0.0.0/8" - if isIPv6 { - loopback = "::1/128" - } - proxier.natRules.Write( - "-A", string(kubeNodePortsChain), - args, - "-s", loopback, "-j", string(KubeMarkMasqChain)) - } - // Jump to the service chain. - proxier.natRules.Write( - "-A", string(kubeNodePortsChain), - args, "-j", string(externalTrafficChain)) } else { // No endpoints. @@ -1347,18 +1339,18 @@ func (proxier *Proxier) syncProxyRules() { } if svcInfo.UsesClusterEndpoints() { - // Write rules jumping from policyClusterChain to clusterEndpoints - proxier.writeServiceToEndpointRules(svcNameString, svcInfo, policyClusterChain, clusterEndpoints, args) + // Write rules jumping from clusterPolicyChain to clusterEndpoints + proxier.writeServiceToEndpointRules(svcNameString, svcInfo, clusterPolicyChain, clusterEndpoints, args) } if svcInfo.UsesLocalEndpoints() { if len(localEndpoints) != 0 { - // Write rules jumping from policyLocalChain to localEndpointChains - proxier.writeServiceToEndpointRules(svcNameString, svcInfo, policyLocalChain, localEndpoints, args) + // Write rules jumping from localPolicyChain to localEndpointChains + proxier.writeServiceToEndpointRules(svcNameString, svcInfo, localPolicyChain, localEndpoints, args) } else if hasEndpoints { // Blackhole all traffic since there are no local endpoints args = append(args[:0], - "-A", string(policyLocalChain), + "-A", string(localPolicyChain), "-m", "comment", "--comment", fmt.Sprintf(`"%s has no local endpoints"`, svcNameString), "-j", @@ -1388,14 +1380,12 @@ func (proxier *Proxier) syncProxyRules() { // Finally, tail-call to the nodeports chain. This needs to be after all // other service portal rules. for address := range nodeAddresses { - // TODO(thockin, m1093782566): If/when we have dual-stack support we will want to distinguish v4 from v6 zero-CIDRs. if utilproxy.IsZeroCIDR(address) { - args = append(args[:0], + proxier.natRules.Write( "-A", string(kubeServicesChain), "-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`, "-m", "addrtype", "--dst-type", "LOCAL", "-j", string(kubeNodePortsChain)) - proxier.natRules.Write(args) // Nothing else matters after the zero CIDR. break } @@ -1405,12 +1395,11 @@ func (proxier *Proxier) syncProxyRules() { continue } // create nodeport rules for each IP one by one - args = append(args[:0], + proxier.natRules.Write( "-A", string(kubeServicesChain), "-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`, "-d", address, "-j", string(kubeNodePortsChain)) - proxier.natRules.Write(args) } // Drop the packets in INVALID state, which would potentially cause @@ -1532,10 +1521,12 @@ func (proxier *Proxier) writeServiceToEndpointRules(svcNameString string, svcInf if !ok { continue } + comment := fmt.Sprintf(`"%s -> %s"`, svcNameString, epInfo.Endpoint) + args = append(args[:0], "-A", string(svcChain), ) - args = proxier.appendServiceCommentLocked(args, svcNameString) + args = proxier.appendServiceCommentLocked(args, comment) args = append(args, "-m", "recent", "--name", string(epInfo.ChainName), "--rcheck", "--seconds", strconv.Itoa(svcInfo.StickyMaxAgeSeconds()), "--reap", @@ -1552,9 +1543,10 @@ func (proxier *Proxier) writeServiceToEndpointRules(svcNameString string, svcInf if !ok { continue } + comment := fmt.Sprintf(`"%s -> %s"`, svcNameString, epInfo.Endpoint) args = append(args[:0], "-A", string(svcChain)) - args = proxier.appendServiceCommentLocked(args, svcNameString) + args = proxier.appendServiceCommentLocked(args, comment) if i < (numEndpoints - 1) { // Each rule is a probabilistic match. args = append(args, diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 3180c92baa6..3544f748b99 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -22,12 +22,15 @@ import ( "net" "reflect" "regexp" + stdruntime "runtime" "sort" "strconv" "strings" "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/lithammer/dedent" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" @@ -455,7 +458,7 @@ func parseIPTablesData(ruleData string) (map[string][]string, error) { return tables, nil } -func Test_parseIPTablesData(t *testing.T) { +func TestParseIPTablesData(t *testing.T) { for _, tc := range []struct { name string input string @@ -464,36 +467,36 @@ func Test_parseIPTablesData(t *testing.T) { }{ { name: "basic test", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + COMMIT + `), output: map[string][]string{ "filter": { `*filter`, @@ -531,122 +534,122 @@ COMMIT }, { name: "not enough tables", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), error: "bad ruleData (1 tables)", }, { name: "trailing junk", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -junk -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + junk + `), error: "bad ruleData (2 tables)", }, { name: "bad start line", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), error: `bad ruleData (table 2 starts with ":KUBE-SERVICES - [0:0]")`, }, { name: "no nat", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*mangle -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *mangle + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), error: `bad ruleData (no "nat" table)`, }, { name: "no filter", - input: ` -*mangle -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -`, + input: dedent.Dedent(` + *mangle + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), error: `bad ruleData (no "filter" table)`, }, } { @@ -772,7 +775,7 @@ func checkIPTablesRuleJumps(ruleData string) error { return nil } -func Test_checkIPTablesRuleJumps(t *testing.T) { +func TestCheckIPTablesRuleJumps(t *testing.T) { for _, tc := range []struct { name string input string @@ -780,110 +783,110 @@ func Test_checkIPTablesRuleJumps(t *testing.T) { }{ { name: "valid", - input: ` -*filter -COMMIT -*nat -:KUBE-MARK-MASQ - [0:0] -:KUBE-SERVICES - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ + COMMIT + `), error: "", }, { name: "can't jump to chain that wasn't created", - input: ` -*filter -COMMIT -*nat -:KUBE-SERVICES - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + COMMIT + `), error: "some chains in nat are used but were not created: [KUBE-SVC-XPGD46QRK7WJZT7O]", }, { name: "can't jump to chain that has no rules", - input: ` -*filter -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + COMMIT + `), error: "some chains in nat are jumped to but have no rules: [KUBE-SVC-XPGD46QRK7WJZT7O]", }, { name: "can't add rules to a chain that wasn't created", - input: ` -*filter -COMMIT -*nat -:KUBE-MARK-MASQ - [0:0] -:KUBE-SERVICES - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" ... --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" ... + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + COMMIT + `), error: "some chains in nat are used but were not created: [KUBE-SVC-XPGD46QRK7WJZT7O]", }, { name: "can't jump to chain that wasn't created", - input: ` -*filter -COMMIT -*nat -:KUBE-SERVICES - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + COMMIT + `), error: "some chains in nat are used but were not created: [KUBE-SVC-XPGD46QRK7WJZT7O]", }, { name: "can't jump to chain that has no rules", - input: ` -*filter -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + COMMIT + `), error: "some chains in nat are jumped to but have no rules: [KUBE-SVC-XPGD46QRK7WJZT7O]", }, { name: "can't add rules to a chain that wasn't created", - input: ` -*filter -COMMIT -*nat -:KUBE-MARK-MASQ - [0:0] -:KUBE-SERVICES - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" ... --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" ... + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + COMMIT + `), error: "some chains in nat are used but were not created: [KUBE-SVC-XPGD46QRK7WJZT7O]", }, { name: "can't create chain and then not use it", - input: ` -*filter -COMMIT -*nat -:KUBE-MARK-MASQ - [0:0] -:KUBE-SERVICES - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" ... -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" ... + COMMIT + `), error: "some chains in nat are created but not used: [KUBE-SVC-XPGD46QRK7WJZT7O]", }, } { @@ -966,7 +969,7 @@ func sortIPTablesRules(ruleData string) (string, error) { return strings.Join(output, "\n"), nil } -func Test_sortIPTablesRules(t *testing.T) { +func TestSortIPTablesRules(t *testing.T) { for _, tc := range []struct { name string input string @@ -975,315 +978,313 @@ func Test_sortIPTablesRules(t *testing.T) { }{ { name: "basic test using each match type", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-XLB-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] -:KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] -:KUBE-SEP-OYPFS5VJICHGATKP - [0:0] -:KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] -:KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] -:KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT --A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment ns2/svc2:p80 -j KUBE-SEP-RS4RBKLTHTF2IUXJ --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 --A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT --A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP --A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-SVC-X27LE4BHSL4DOUIK --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-MARK-MASQ --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -j KUBE-SEP-OYPFS5VJICHGATKP --A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ --A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j KUBE-SEP-C6EBXVWJJZMIWKLZ --A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ --A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 --A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ --A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, - output: ` -*filter -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-SERVICES - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] -:KUBE-SEP-OYPFS5VJICHGATKP - [0:0] -:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] -:KUBE-SERVICES - [0:0] -:KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] -:KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-XLB-GNZBNJ2PO5MGZ6GT - [0:0] --A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-SVC-X27LE4BHSL4DOUIK --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT --A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK --A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ --A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 --A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ --A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ --A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j KUBE-SEP-C6EBXVWJJZMIWKLZ --A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment ns2/svc2:p80 -j KUBE-SEP-RS4RBKLTHTF2IUXJ --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-MARK-MASQ --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -j KUBE-SEP-OYPFS5VJICHGATKP --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT -COMMIT -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-EXT-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] + :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] + :KUBE-SEP-OYPFS5VJICHGATKP - [0:0] + :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] + :KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] + :KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment ns2/svc2:p80 -j KUBE-SEP-RS4RBKLTHTF2IUXJ + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP + -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT + -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-SVC-X27LE4BHSL4DOUIK + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-MARK-MASQ + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -j KUBE-SEP-OYPFS5VJICHGATKP + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j KUBE-SEP-C6EBXVWJJZMIWKLZ + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + COMMIT + `), + output: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] + :KUBE-SEP-OYPFS5VJICHGATKP - [0:0] + :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] + :KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-SVC-X27LE4BHSL4DOUIK + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j KUBE-SEP-C6EBXVWJJZMIWKLZ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment ns2/svc2:p80 -j KUBE-SEP-RS4RBKLTHTF2IUXJ + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-MARK-MASQ + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -j KUBE-SEP-OYPFS5VJICHGATKP + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP + COMMIT + `), }, { name: "not enough tables", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), error: "bad ruleData (1 tables)", }, { name: "extra tables", - input: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*mangle -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -`, - output: ` -*filter -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-SERVICES - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*mangle -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-SERVICES - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-SERVICES - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -`, + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *mangle + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), + output: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *mangle + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), }, { name: "correctly match same service name in different styles of comments", - input: ` -*filter -COMMIT -*nat -:KUBE-SERVICES - [0:0] --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 --A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 --A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" svc1 line 1 --A KUBE-SERVICES -m comment --comment ns1/svc1 svc1 line 2 --A KUBE-SERVICES -m comment --comment "ns1/svc1 blah" svc1 line 3 --A KUBE-SERVICES -m comment --comment ns4/svc4 svc4 line 1 --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" svc4 line 2 --A KUBE-SERVICES -m comment --comment "ns4/svc4 blah" svc4 line 3 --A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" svc3 line 1 --A KUBE-SERVICES -m comment --comment "ns3/svc3 blah" svc3 line 2 --A KUBE-SERVICES -m comment --comment ns3/svc3 svc3 line 3 -COMMIT -`, - output: ` -*filter -COMMIT -*nat -:KUBE-SERVICES - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" svc1 line 1 --A KUBE-SERVICES -m comment --comment ns1/svc1 svc1 line 2 --A KUBE-SERVICES -m comment --comment "ns1/svc1 blah" svc1 line 3 --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 --A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 --A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 --A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" svc3 line 1 --A KUBE-SERVICES -m comment --comment "ns3/svc3 blah" svc3 line 2 --A KUBE-SERVICES -m comment --comment ns3/svc3 svc3 line 3 --A KUBE-SERVICES -m comment --comment ns4/svc4 svc4 line 1 --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" svc4 line 2 --A KUBE-SERVICES -m comment --comment "ns4/svc4 blah" svc4 line 3 -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 + -A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 + -A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" svc1 line 1 + -A KUBE-SERVICES -m comment --comment ns1/svc1 svc1 line 2 + -A KUBE-SERVICES -m comment --comment "ns1/svc1 blah" svc1 line 3 + -A KUBE-SERVICES -m comment --comment ns4/svc4 svc4 line 1 + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" svc4 line 2 + -A KUBE-SERVICES -m comment --comment "ns4/svc4 blah" svc4 line 3 + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" svc3 line 1 + -A KUBE-SERVICES -m comment --comment "ns3/svc3 blah" svc3 line 2 + -A KUBE-SERVICES -m comment --comment ns3/svc3 svc3 line 3 + COMMIT + `), + output: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" svc1 line 1 + -A KUBE-SERVICES -m comment --comment ns1/svc1 svc1 line 2 + -A KUBE-SERVICES -m comment --comment "ns1/svc1 blah" svc1 line 3 + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 + -A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 + -A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" svc3 line 1 + -A KUBE-SERVICES -m comment --comment "ns3/svc3 blah" svc3 line 2 + -A KUBE-SERVICES -m comment --comment ns3/svc3 svc3 line 3 + -A KUBE-SERVICES -m comment --comment ns4/svc4 svc4 line 1 + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" svc4 line 2 + -A KUBE-SERVICES -m comment --comment "ns4/svc4 blah" svc4 line 3 + COMMIT + `), }, { name: "unexpected junk lines are preserved", - input: ` -*filter -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] -:KUBE-AAAAA - [0:0] -:KUBE-ZZZZZ - [0:0] -:WHY-IS-THIS-CHAIN-HERE - [0:0] --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 --A KUBE-ZZZZZ -m comment --comment "mystery chain number 1" --A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 --A WHY-IS-THIS-CHAIN-HERE -j ACCEPT --A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 --A KUBE-AAAAA -m comment --comment "mystery chain number 2" -COMMIT -`, - output: ` -*filter -COMMIT -*nat -:KUBE-AAAAA - [0:0] -:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] -:KUBE-SERVICES - [0:0] -:KUBE-ZZZZZ - [0:0] -:WHY-IS-THIS-CHAIN-HERE - [0:0] --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 --A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 --A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 --A KUBE-AAAAA -m comment --comment "mystery chain number 2" --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 --A KUBE-ZZZZZ -m comment --comment "mystery chain number 1" --A WHY-IS-THIS-CHAIN-HERE -j ACCEPT -COMMIT -`, + input: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] + :KUBE-AAAAA - [0:0] + :KUBE-ZZZZZ - [0:0] + :WHY-IS-THIS-CHAIN-HERE - [0:0] + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 + -A KUBE-ZZZZZ -m comment --comment "mystery chain number 1" + -A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 + -A WHY-IS-THIS-CHAIN-HERE -j ACCEPT + -A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 + -A KUBE-AAAAA -m comment --comment "mystery chain number 2" + COMMIT + `), + output: dedent.Dedent(` + *filter + COMMIT + *nat + :KUBE-AAAAA - [0:0] + :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-ZZZZZ - [0:0] + :WHY-IS-THIS-CHAIN-HERE - [0:0] + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 + -A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 + -A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 + -A KUBE-AAAAA -m comment --comment "mystery chain number 2" + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 + -A KUBE-ZZZZZ -m comment --comment "mystery chain number 1" + -A WHY-IS-THIS-CHAIN-HERE -j ACCEPT + COMMIT + `), }, } { t.Run(tc.name, func(t *testing.T) { @@ -1305,19 +1306,34 @@ COMMIT } } +// getLine returns the line number of the caller, if possible. This is useful in +// tests with a large number of cases - when something goes wrong you can find +// which case more easily. +func getLine() int { + _, _, line, ok := stdruntime.Caller(1) + if ok { + return line + } + return 0 +} + // assertIPTablesRulesEqual asserts that the generated rules in result match the rules in // expected, ignoring irrelevant ordering differences. -func assertIPTablesRulesEqual(t *testing.T, expected, result string) { - expected, err := sortIPTablesRules(expected) - if err != nil { - t.Fatalf("%s", err) - } - result, err = sortIPTablesRules(result) +func assertIPTablesRulesEqual(t *testing.T, line int, expected, result string) { + expected = strings.TrimLeft(expected, " \t\n") + + result, err := sortIPTablesRules(strings.TrimLeft(result, " \t\n")) if err != nil { t.Fatalf("%s", err) } - assert.Equal(t, expected, result) + lineStr := "" + if line != 0 { + lineStr = fmt.Sprintf(" (from line %d)", line) + } + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("rules do not match%s:\ndiff:\n%s\nfull result:\n```\n%s```", lineStr, diff, result) + } err = checkIPTablesRuleJumps(expected) if err != nil { @@ -1327,17 +1343,21 @@ func assertIPTablesRulesEqual(t *testing.T, expected, result string) { // assertIPTablesRulesNotEqual asserts that the generated rules in result DON'T match the // rules in expected, ignoring irrelevant ordering differences. -func assertIPTablesRulesNotEqual(t *testing.T, expected, result string) { - expected, err := sortIPTablesRules(expected) - if err != nil { - t.Fatalf("%s", err) - } - result, err = sortIPTablesRules(result) +func assertIPTablesRulesNotEqual(t *testing.T, line int, expected, result string) { + expected = strings.TrimLeft(expected, " \t\n") + + result, err := sortIPTablesRules(strings.TrimLeft(result, " \t\n")) if err != nil { t.Fatalf("%s", err) } - assert.NotEqual(t, expected, result) + lineStr := "" + if line != 0 { + lineStr = fmt.Sprintf(" (from line %d)", line) + } + if cmp.Equal(expected, result) { + t.Errorf("rules do not differ%s:\nfull result:\n```\n%s```", lineStr, result) + } err = checkIPTablesRuleJumps(expected) if err != nil { @@ -1498,94 +1518,96 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-XLB-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] -:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] -:KUBE-SVC-PAZTZYUUMV5KCDZL - [0:0] -:KUBE-FW-PAZTZYUUMV5KCDZL - [0:0] -:KUBE-SEP-QDCEFMBQEGWIV4VT - [0:0] -:KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] -:KUBE-SEP-OYPFS5VJICHGATKP - [0:0] -:KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] -:KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] -:KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT --A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment ns2/svc2:p80 -j KUBE-SEP-RS4RBKLTHTF2IUXJ --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ --A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 --A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT --A KUBE-XLB-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT --A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP --A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-PAZTZYUUMV5KCDZL --A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -m tcp -p tcp -d 5.6.7.8 --dport 80 -j KUBE-FW-PAZTZYUUMV5KCDZL --A KUBE-FW-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -j KUBE-MARK-MASQ --A KUBE-FW-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -j KUBE-SVC-PAZTZYUUMV5KCDZL --A KUBE-FW-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp --dport 3002 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp --dport 3002 -j KUBE-SVC-PAZTZYUUMV5KCDZL --A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment ns2b/svc2b:p80 -j KUBE-SEP-QDCEFMBQEGWIV4VT --A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ --A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 --A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3003 -j KUBE-SVC-X27LE4BHSL4DOUIK --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3003 -j KUBE-MARK-MASQ --A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -j KUBE-SEP-OYPFS5VJICHGATKP --A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ --A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK --A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC --A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j KUBE-SEP-C6EBXVWJJZMIWKLZ --A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ --A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 --A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ --A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-4SW47YFZTEDKD3PK - [0:0] + :KUBE-EXT-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-EXT-PAZTZYUUMV5KCDZL - [0:0] + :KUBE-EXT-X27LE4BHSL4DOUIK - [0:0] + :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] + :KUBE-SEP-OYPFS5VJICHGATKP - [0:0] + :KUBE-SEP-QDCEFMBQEGWIV4VT - [0:0] + :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] + :KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-SVC-PAZTZYUUMV5KCDZL - [0:0] + :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-NODEPORTS -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp --dport 3002 -j KUBE-EXT-PAZTZYUUMV5KCDZL + -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3003 -j KUBE-EXT-X27LE4BHSL4DOUIK + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-PAZTZYUUMV5KCDZL + -A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -m tcp -p tcp -d 5.6.7.8 --dport 80 -j KUBE-EXT-PAZTZYUUMV5KCDZL + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 -j KUBE-EXT-4SW47YFZTEDKD3PK + -A KUBE-EXT-4SW47YFZTEDKD3PK -m comment --comment "masquerade traffic for ns4/svc4:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-4SW47YFZTEDKD3PK -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "pod traffic for ns2/svc2:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-PAZTZYUUMV5KCDZL -m comment --comment "masquerade traffic for ns2b/svc2b:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-PAZTZYUUMV5KCDZL -j KUBE-SVC-PAZTZYUUMV5KCDZL + -A KUBE-EXT-X27LE4BHSL4DOUIK -m comment --comment "masquerade traffic for ns3/svc3:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-X27LE4BHSL4DOUIK -j KUBE-SVC-X27LE4BHSL4DOUIK + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 + -A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.4:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.5:80" -j KUBE-SEP-C6EBXVWJJZMIWKLZ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 -> 10.180.0.2:80" -j KUBE-SEP-RS4RBKLTHTF2IUXJ + -A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 -> 10.180.0.3:80" -j KUBE-SEP-QDCEFMBQEGWIV4VT + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 -> 10.180.0.3:80" -j KUBE-SEP-OYPFS5VJICHGATKP + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP + COMMIT + `) + + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) natRulesMetric, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) if err != nil { @@ -1622,31 +1644,31 @@ func TestClusterIPReject(t *testing.T) { ) fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `) - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestClusterIPEndpointsJump(t *testing.T) { @@ -1689,37 +1711,36 @@ func TestClusterIPEndpointsJump(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestLoadBalancer(t *testing.T) { @@ -1772,44 +1793,45 @@ func TestLoadBalancer(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-FW-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-MASQ --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-FW-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-DROP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ + COMMIT + `) - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestNodePort(t *testing.T) { @@ -1855,38 +1877,40 @@ func TestNodePort(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestHealthCheckNodePort(t *testing.T) { @@ -1921,33 +1945,33 @@ func TestHealthCheckNodePort(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 127.0.0.1 -j KUBE-NODEPORTS -COMMIT -` + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 127.0.0.1 -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `) - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestMasqueradeRule(t *testing.T) { @@ -1956,35 +1980,35 @@ func TestMasqueradeRule(t *testing.T) { fp := NewFakeProxier(ipt) fp.syncProxyRules() - expectedFmt := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE%s --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` + expectedFmt := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE%s + COMMIT + `) var expected string if testcase { expected = fmt.Sprintf(expectedFmt, " --random-fully") } else { expected = fmt.Sprintf(expectedFmt, "") } - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } } @@ -2015,32 +2039,31 @@ func TestExternalIPsReject(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j REJECT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestOnlyLocalExternalIPs(t *testing.T) { @@ -2090,49 +2113,48 @@ func TestOnlyLocalExternalIPs(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] -:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-XLB-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-XLB-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O --A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } // TestNonLocalExternalIPs tests if we add the masquerade rule into svcChain in order to @@ -2183,43 +2205,44 @@ func TestNonLocalExternalIPs(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestNodePortReject(t *testing.T) { @@ -2248,32 +2271,31 @@ func TestNodePortReject(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestLoadBalancerReject(t *testing.T) { @@ -2314,34 +2336,33 @@ func TestLoadBalancerReject(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestOnlyLocalLoadBalancing(t *testing.T) { @@ -2403,58 +2424,53 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] -:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] -:KUBE-FW-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-XLB-XPGD46QRK7WJZT7O --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-XLB-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O --A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { @@ -2463,47 +2479,46 @@ func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() fp.nodePortAddresses = []string{"192.168.0.0/24"} - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] -:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-XLB-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O --A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS -COMMIT -` - onlyLocalNodePorts(t, fp, ipt, expected) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + COMMIT + `) + onlyLocalNodePorts(t, fp, ipt, expected, getLine()) } func TestOnlyLocalNodePorts(t *testing.T) { @@ -2511,52 +2526,51 @@ func TestOnlyLocalNodePorts(t *testing.T) { fp := NewFakeProxier(ipt) fp.nodePortAddresses = []string{"192.168.0.0/24"} - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] -:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] -:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ --A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-XLB-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O --A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS -COMMIT -` - onlyLocalNodePorts(t, fp, ipt, expected) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ + COMMIT + `) + onlyLocalNodePorts(t, fp, ipt, expected, getLine()) } -func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTables, expected string) { +func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTables, expected string, line int) { svcIP := "172.30.0.41" svcPort := 80 svcNodePort := 3001 @@ -2603,7 +2617,7 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable fp.syncProxyRules() - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, line, expected, fp.iptablesData.String()) } func TestComputeProbability(t *testing.T) { @@ -2971,7 +2985,7 @@ func compareEndpointsMapsExceptChainName(t *testing.T, tci int, newMap proxy.End } } -func Test_updateEndpointsMap(t *testing.T) { +func TestUpdateEndpointsMap(t *testing.T) { var nodeName = testHostname udpProtocol := v1.ProtocolUDP @@ -3790,43 +3804,43 @@ func Test_updateEndpointsMap(t *testing.T) { // the shared EndpointChangeTracker and EndpointSliceCache. This test ensures that the // iptables proxier supports translating EndpointSlices to iptables output. func TestEndpointSliceE2E(t *testing.T) { - expectedIPTablesWithSlice := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` + expectedIPTablesWithSlice := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT + COMMIT + `) ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) @@ -3879,61 +3893,60 @@ COMMIT fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - assertIPTablesRulesEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, getLine(), expectedIPTablesWithSlice, fp.iptablesData.String()) fp.OnEndpointSliceDelete(endpointSlice) fp.syncProxyRules() - assertIPTablesRulesNotEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, getLine(), expectedIPTablesWithSlice, fp.iptablesData.String()) } func TestHealthCheckNodePortE2E(t *testing.T) { - expectedIPTables := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-NODEPORTS -m comment --comment ns1/svc1 -m tcp -p tcp --dport 30010 -s 127.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns1/svc1 -m tcp -p tcp --dport 30010 -j KUBE-XLB-AQI2S6QIMU7PVVRP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` + expectedIPTables := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns1/svc1 -m tcp -p tcp --dport 30010 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -j KUBE-SEP-3JOIVZTXZZRGORX4 + COMMIT + `) ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) @@ -3989,15 +4002,15 @@ COMMIT } fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - assertIPTablesRulesEqual(t, expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, getLine(), expectedIPTables, fp.iptablesData.String()) fp.OnServiceDelete(svc) fp.syncProxyRules() - assertIPTablesRulesNotEqual(t, expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, getLine(), expectedIPTables, fp.iptablesData.String()) } -// Test_HealthCheckNodePortWhenTerminating tests that health check node ports are not enabled when all local endpoints are terminating -func Test_HealthCheckNodePortWhenTerminating(t *testing.T) { +// TestHealthCheckNodePortWhenTerminating tests that health check node ports are not enabled when all local endpoints are terminating +func TestHealthCheckNodePortWhenTerminating(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) fp.OnServiceSynced() @@ -4349,46 +4362,47 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { cluster := v1.ServiceInternalTrafficPolicyCluster local := v1.ServiceInternalTrafficPolicyLocal - clusterExpectedIPTables := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` + clusterExpectedIPTables := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT + COMMIT + `) testCases := []struct { name string + line int internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType featureGateOn bool endpoints []endpoint @@ -4397,6 +4411,7 @@ COMMIT }{ { name: "internalTrafficPolicy is cluster", + line: getLine(), internalTrafficPolicy: &cluster, featureGateOn: true, endpoints: []endpoint{ @@ -4409,6 +4424,7 @@ COMMIT }, { name: "internalTrafficPolicy is local and there is non-zero local endpoints", + line: getLine(), internalTrafficPolicy: &local, featureGateOn: true, endpoints: []endpoint{ @@ -4417,38 +4433,39 @@ COMMIT {"10.0.1.3", "host2"}, }, expectEndpointRule: true, - expectedIPTablesWithSlice: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTablesWithSlice: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -j KUBE-SEP-3JOIVZTXZZRGORX4 + COMMIT + `), }, { name: "internalTrafficPolicy is local and there is zero local endpoint", + line: getLine(), internalTrafficPolicy: &local, featureGateOn: true, endpoints: []endpoint{ @@ -4457,35 +4474,36 @@ COMMIT {"10.0.1.3", "host2"}, }, expectEndpointRule: false, - expectedIPTablesWithSlice: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTablesWithSlice: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP + COMMIT + `), }, { name: "internalTrafficPolicy is local and there is non-zero local endpoint with feature gate off", + line: getLine(), internalTrafficPolicy: &local, featureGateOn: false, endpoints: []endpoint{ @@ -4547,20 +4565,20 @@ COMMIT fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - assertIPTablesRulesEqual(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, tc.line, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) if tc.expectEndpointRule { fp.OnEndpointSliceDelete(endpointSlice) fp.syncProxyRules() - assertIPTablesRulesNotEqual(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, tc.line, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) } }) } } -// Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyLocal tests that when there are local ready and ready + terminating +// TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal tests that when there are local ready and ready + terminating // endpoints, only the ready endpoints are used. -func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { +func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { tcpProtocol := v1.ProtocolTCP timeout := v1.DefaultClientIPServiceAffinitySeconds service := &v1.Service{ @@ -4597,6 +4615,7 @@ func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) testcases := []struct { name string + line int terminatingFeatureGate bool endpointslice *discovery.EndpointSlice expectedIPTables string @@ -4604,6 +4623,7 @@ func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -4668,64 +4688,62 @@ func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -j KUBE-SEP-IO5XOSKPAXIFQXAJ + COMMIT + `), }, { name: "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist", + line: getLine(), terminatingFeatureGate: false, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -4790,64 +4808,62 @@ COMMIT }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -j KUBE-SEP-IO5XOSKPAXIFQXAJ + COMMIT + `), }, { name: "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -4904,60 +4920,58 @@ COMMIT }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT + COMMIT + `), }, { name: "with ProxyTerminatingEndpoints disabled, only terminating endpoints exist", + line: getLine(), terminatingFeatureGate: false, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5019,51 +5033,49 @@ COMMIT }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP + COMMIT + `), }, { name: "ProxyTerminatingEndpoints enabled, terminating endpoints on remote node", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5091,51 +5103,49 @@ COMMIT }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ --A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-XLB-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP --A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP + COMMIT + `), }, { name: "no usable endpoints on any node", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5173,31 +5183,31 @@ COMMIT }, }, noUsableEndpoints: true, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT --A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `), }, } @@ -5214,24 +5224,23 @@ COMMIT fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() - t.Log(fp.iptablesData.String()) - assertIPTablesRulesEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() if testcase.noUsableEndpoints { // Deleting the EndpointSlice should have had no effect - assertIPTablesRulesEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } else { - assertIPTablesRulesNotEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } }) } } -// Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyCluster tests that when there are cluster-wide ready and ready + terminating +// TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster tests that when there are cluster-wide ready and ready + terminating // endpoints, only the ready endpoints are used. -func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { +func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { tcpProtocol := v1.ProtocolTCP timeout := v1.DefaultClientIPServiceAffinitySeconds service := &v1.Service{ @@ -5268,6 +5277,7 @@ func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T testcases := []struct { name string + line int terminatingFeatureGate bool endpointslice *discovery.EndpointSlice expectedIPTables string @@ -5275,6 +5285,7 @@ func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5338,54 +5349,54 @@ func Test_EndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-MASQ --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + COMMIT + `), }, { name: "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist", + line: getLine(), terminatingFeatureGate: false, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5449,54 +5460,54 @@ COMMIT }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-MASQ --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + COMMIT + `), }, { name: "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5553,54 +5564,54 @@ COMMIT }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-MASQ --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] + :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-IO5XOSKPAXIFQXAJ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-XGJFVO3L2O5SRFNT + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + COMMIT + `), }, { name: "with ProxyTerminatingEndpoints disabled, only terminating endpoints exist", + line: getLine(), terminatingFeatureGate: false, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5663,33 +5674,34 @@ COMMIT }, }, noUsableEndpoints: true, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `), }, { name: "ProxyTerminatingEndpoints enabled, terminating endpoints on remote node", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5715,44 +5727,44 @@ COMMIT }, }, }, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] -:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-MASQ --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-SVC-AQI2S6QIMU7PVVRP --A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY + -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY + COMMIT + `), }, { name: "no usable endpoints on any node", + line: getLine(), terminatingFeatureGate: true, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -5790,30 +5802,30 @@ COMMIT }, }, noUsableEndpoints: true, - expectedIPTables: ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT --A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -`, + expectedIPTables: dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + COMMIT + `), }, } @@ -5830,16 +5842,15 @@ COMMIT fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() - t.Log(fp.iptablesData.String()) - assertIPTablesRulesEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() if testcase.noUsableEndpoints { // Deleting the EndpointSlice should have had no effect - assertIPTablesRulesEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } else { - assertIPTablesRulesNotEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } }) } @@ -5883,44 +5894,41 @@ func TestMasqueradeAll(t *testing.T) { fp.syncProxyRules() - expected := ` -*filter -:KUBE-SERVICES - [0:0] -:KUBE-EXTERNAL-SERVICES - [0:0] -:KUBE-FORWARD - [0:0] -:KUBE-NODEPORTS - [0:0] --A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP --A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT --A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -COMMIT -*nat -:KUBE-SERVICES - [0:0] -:KUBE-NODEPORTS - [0:0] -:KUBE-POSTROUTING - [0:0] -:KUBE-MARK-MASQ - [0:0] -:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -:KUBE-FW-XPGD46QRK7WJZT7O - [0:0] -:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] --A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN --A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 --A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-MARK-MASQ --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-MASQ --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ --A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-XPGD46QRK7WJZT7O --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ --A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 --A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -COMMIT -` - - assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) + expected := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ + COMMIT + `) + assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) } func countEndpointsAndComments(iptablesData string, matchEndpoint string) (string, int, int) { diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 635f30f9d41..fe91de9691f 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -1202,7 +1202,7 @@ func (proxier *Proxier) syncProxyRules() { // ExternalTrafficPolicy only works for NodePort and external LB traffic, does not affect ClusterIP // So we still need clusterIP rules in onlyNodeLocalEndpoints mode. internalNodeLocal := false - if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) && svcInfo.NodeLocalInternal() { + if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) && svcInfo.InternalPolicyLocal() { internalNodeLocal = true } if err := proxier.syncEndpoint(svcName, internalNodeLocal, serv); err != nil { @@ -1222,7 +1222,7 @@ func (proxier *Proxier) syncProxyRules() { SetType: utilipset.HashIPPort, } - if svcInfo.NodeLocalExternal() { + if svcInfo.ExternalPolicyLocal() { if valid := proxier.ipsetList[kubeExternalIPLocalSet].validateEntry(entry); !valid { klog.ErrorS(nil, "Error adding entry to ipset", "entry", entry, "ipset", proxier.ipsetList[kubeExternalIPLocalSet].Name) continue @@ -1252,7 +1252,7 @@ func (proxier *Proxier) syncProxyRules() { activeIPVSServices[serv.String()] = true activeBindAddrs[serv.Address.String()] = true - if err := proxier.syncEndpoint(svcName, svcInfo.NodeLocalExternal(), serv); err != nil { + if err := proxier.syncEndpoint(svcName, svcInfo.ExternalPolicyLocal(), serv); err != nil { klog.ErrorS(err, "Failed to sync endpoint for service", "serviceName", svcName, "virtualServer", serv) } } else { @@ -1279,7 +1279,7 @@ func (proxier *Proxier) syncProxyRules() { } proxier.ipsetList[kubeLoadBalancerSet].activeEntries.Insert(entry.String()) // insert loadbalancer entry to lbIngressLocalSet if service externaltrafficpolicy=local - if svcInfo.NodeLocalExternal() { + if svcInfo.ExternalPolicyLocal() { if valid := proxier.ipsetList[kubeLoadBalancerLocalSet].validateEntry(entry); !valid { klog.ErrorS(nil, "Error adding entry to ipset", "entry", entry, "ipset", proxier.ipsetList[kubeLoadBalancerLocalSet].Name) continue @@ -1351,7 +1351,7 @@ func (proxier *Proxier) syncProxyRules() { if err := proxier.syncService(svcNameString, serv, true, bindedAddresses); err == nil { activeIPVSServices[serv.String()] = true activeBindAddrs[serv.Address.String()] = true - if err := proxier.syncEndpoint(svcName, svcInfo.NodeLocalExternal(), serv); err != nil { + if err := proxier.syncEndpoint(svcName, svcInfo.ExternalPolicyLocal(), serv); err != nil { klog.ErrorS(err, "Failed to sync endpoint for service", "serviceName", svcName, "virtualServer", serv) } } else { @@ -1449,7 +1449,7 @@ func (proxier *Proxier) syncProxyRules() { } // Add externaltrafficpolicy=local type nodeport entry - if svcInfo.NodeLocalExternal() { + if svcInfo.ExternalPolicyLocal() { var nodePortLocalSet *IPSet switch protocol { case utilipset.ProtocolTCP: @@ -1494,7 +1494,7 @@ func (proxier *Proxier) syncProxyRules() { // There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`. if err := proxier.syncService(svcNameString, serv, false, bindedAddresses); err == nil { activeIPVSServices[serv.String()] = true - if err := proxier.syncEndpoint(svcName, svcInfo.NodeLocalExternal(), serv); err != nil { + if err := proxier.syncEndpoint(svcName, svcInfo.ExternalPolicyLocal(), serv); err != nil { klog.ErrorS(err, "Failed to sync endpoint for service", "serviceName", svcName, "virtualServer", serv) } } else { diff --git a/pkg/proxy/service.go b/pkg/proxy/service.go index 71b141a3376..17e746fbfac 100644 --- a/pkg/proxy/service.go +++ b/pkg/proxy/service.go @@ -52,8 +52,8 @@ type BaseServiceInfo struct { externalIPs []string loadBalancerSourceRanges []string healthCheckNodePort int - nodeLocalExternal bool - nodeLocalInternal bool + externalPolicyLocal bool + internalPolicyLocal bool internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType hintsAnnotation string } @@ -119,14 +119,14 @@ func (info *BaseServiceInfo) LoadBalancerIPStrings() []string { return ips } -// NodeLocalExternal is part of ServicePort interface. -func (info *BaseServiceInfo) NodeLocalExternal() bool { - return info.nodeLocalExternal +// ExternalPolicyLocal is part of ServicePort interface. +func (info *BaseServiceInfo) ExternalPolicyLocal() bool { + return info.externalPolicyLocal } -// NodeLocalInternal is part of ServicePort interface -func (info *BaseServiceInfo) NodeLocalInternal() bool { - return info.nodeLocalInternal +// InternalPolicyLocal is part of ServicePort interface +func (info *BaseServiceInfo) InternalPolicyLocal() bool { + return info.internalPolicyLocal } // InternalTrafficPolicy is part of ServicePort interface @@ -149,22 +149,22 @@ func (info *BaseServiceInfo) UsesClusterEndpoints() bool { // The service port uses Cluster endpoints if the internal traffic policy is "Cluster", // or if it accepts external traffic at all. (Even if the external traffic policy is // "Local", we need Cluster endpoints to implement short circuiting.) - return !info.nodeLocalInternal || info.ExternallyAccessible() + return !info.internalPolicyLocal || info.ExternallyAccessible() } // UsesLocalEndpoints is part of ServicePort interface. func (info *BaseServiceInfo) UsesLocalEndpoints() bool { - return info.nodeLocalInternal || (info.nodeLocalExternal && info.ExternallyAccessible()) + return info.internalPolicyLocal || (info.externalPolicyLocal && info.ExternallyAccessible()) } func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, service *v1.Service) *BaseServiceInfo { - nodeLocalExternal := false - if apiservice.RequestsOnlyLocalTraffic(service) { - nodeLocalExternal = true + externalPolicyLocal := false + if apiservice.ExternalPolicyLocal(service) { + externalPolicyLocal = true } - nodeLocalInternal := false + internalPolicyLocal := false if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) { - nodeLocalInternal = apiservice.RequestsOnlyLocalTrafficForInternal(service) + internalPolicyLocal = apiservice.InternalPolicyLocal(service) } var stickyMaxAgeSeconds int if service.Spec.SessionAffinity == v1.ServiceAffinityClientIP { @@ -180,8 +180,8 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic nodePort: int(port.NodePort), sessionAffinityType: service.Spec.SessionAffinity, stickyMaxAgeSeconds: stickyMaxAgeSeconds, - nodeLocalExternal: nodeLocalExternal, - nodeLocalInternal: nodeLocalInternal, + externalPolicyLocal: externalPolicyLocal, + internalPolicyLocal: internalPolicyLocal, internalTrafficPolicy: service.Spec.InternalTrafficPolicy, hintsAnnotation: service.Annotations[v1.AnnotationTopologyAwareHints], } diff --git a/pkg/proxy/topology_test.go b/pkg/proxy/topology_test.go index cdab4926ccb..77087a66bbf 100644 --- a/pkg/proxy/topology_test.go +++ b/pkg/proxy/topology_test.go @@ -120,7 +120,7 @@ func TestCategorizeEndpoints(t *testing.T) { name: "externalTrafficPolicy: Local, topology ignored for Local endpoints", hintsEnabled: true, nodeLabels: map[string]string{v1.LabelTopologyZone: "zone-a"}, - serviceInfo: &BaseServiceInfo{nodeLocalExternal: true, nodePort: 8080, hintsAnnotation: "auto"}, + serviceInfo: &BaseServiceInfo{externalPolicyLocal: true, nodePort: 8080, hintsAnnotation: "auto"}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.1.2.3:80", ZoneHints: sets.NewString("zone-a"), Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.1.2.4:80", ZoneHints: sets.NewString("zone-b"), Ready: true, IsLocal: true}, @@ -134,7 +134,7 @@ func TestCategorizeEndpoints(t *testing.T) { name: "internalTrafficPolicy: Local, topology ignored for Local endpoints", hintsEnabled: true, nodeLabels: map[string]string{v1.LabelTopologyZone: "zone-a"}, - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true, hintsAnnotation: "auto", nodeLocalExternal: false, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true, hintsAnnotation: "auto", externalPolicyLocal: false, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.1.2.3:80", ZoneHints: sets.NewString("zone-a"), Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.1.2.4:80", ZoneHints: sets.NewString("zone-b"), Ready: true, IsLocal: true}, @@ -282,7 +282,7 @@ func TestCategorizeEndpoints(t *testing.T) { name: "conflicting topology and localness require merging allEndpoints", hintsEnabled: true, nodeLabels: map[string]string{v1.LabelTopologyZone: "zone-a"}, - serviceInfo: &BaseServiceInfo{nodeLocalInternal: false, nodeLocalExternal: true, nodePort: 8080, hintsAnnotation: "auto"}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: false, externalPolicyLocal: true, nodePort: 8080, hintsAnnotation: "auto"}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", ZoneHints: sets.NewString("zone-a"), Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", ZoneHints: sets.NewString("zone-b"), Ready: true, IsLocal: true}, @@ -294,13 +294,13 @@ func TestCategorizeEndpoints(t *testing.T) { allEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.1:80", "10.0.0.2:80"), }, { name: "internalTrafficPolicy: Local, with empty endpoints", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true}, endpoints: []Endpoint{}, clusterEndpoints: nil, localEndpoints: sets.NewString(), }, { name: "internalTrafficPolicy: Local, but all endpoints are remote", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: false}, @@ -310,7 +310,7 @@ func TestCategorizeEndpoints(t *testing.T) { onlyRemoteEndpoints: true, }, { name: "internalTrafficPolicy: Local, all endpoints are local", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: true}, @@ -319,7 +319,7 @@ func TestCategorizeEndpoints(t *testing.T) { localEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.1:80"), }, { name: "internalTrafficPolicy: Local, some endpoints are local", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: false}, @@ -366,7 +366,7 @@ func TestCategorizeEndpoints(t *testing.T) { localEndpoints: nil, }, { name: "iTP: Local, eTP: Cluster, some endpoints local", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true, nodeLocalExternal: false, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true, externalPolicyLocal: false, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: false}, @@ -376,7 +376,7 @@ func TestCategorizeEndpoints(t *testing.T) { allEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.1:80"), }, { name: "iTP: Cluster, eTP: Local, some endpoints local", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: false, nodeLocalExternal: true, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: false, externalPolicyLocal: true, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: false}, @@ -386,7 +386,7 @@ func TestCategorizeEndpoints(t *testing.T) { allEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.1:80"), }, { name: "iTP: Local, eTP: Local, some endpoints local", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true, nodeLocalExternal: true, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true, externalPolicyLocal: true, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: true}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: false}, @@ -396,7 +396,7 @@ func TestCategorizeEndpoints(t *testing.T) { allEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.1:80"), }, { name: "iTP: Local, eTP: Local, all endpoints remote", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true, nodeLocalExternal: true, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true, externalPolicyLocal: true, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: false}, @@ -406,7 +406,7 @@ func TestCategorizeEndpoints(t *testing.T) { allEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.1:80"), }, { name: "iTP: Local, eTP: Local, PTE disabled, all endpoints remote and terminating", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true, nodeLocalExternal: true, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true, externalPolicyLocal: true, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: false, Serving: true, Terminating: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: false, Serving: true, Terminating: true, IsLocal: false}, @@ -417,7 +417,7 @@ func TestCategorizeEndpoints(t *testing.T) { }, { name: "iTP: Local, eTP: Local, PTE enabled, all endpoints remote and terminating", pteEnabled: true, - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true, nodeLocalExternal: true, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true, externalPolicyLocal: true, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: false, Serving: true, Terminating: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: false, Serving: true, Terminating: true, IsLocal: false}, @@ -428,7 +428,7 @@ func TestCategorizeEndpoints(t *testing.T) { onlyRemoteEndpoints: true, }, { name: "iTP: Cluster, eTP: Local, PTE disabled, with terminating endpoints", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: false, nodeLocalExternal: true, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: false, externalPolicyLocal: true, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: false, Serving: false, IsLocal: true}, @@ -441,7 +441,7 @@ func TestCategorizeEndpoints(t *testing.T) { }, { name: "iTP: Cluster, eTP: Local, PTE enabled, with terminating endpoints", pteEnabled: true, - serviceInfo: &BaseServiceInfo{nodeLocalInternal: false, nodeLocalExternal: true, nodePort: 8080}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: false, externalPolicyLocal: true, nodePort: 8080}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: false, Serving: false, IsLocal: true}, @@ -453,7 +453,7 @@ func TestCategorizeEndpoints(t *testing.T) { allEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.2:80"), }, { name: "externalTrafficPolicy ignored if not externally accessible", - serviceInfo: &BaseServiceInfo{nodeLocalExternal: true}, + serviceInfo: &BaseServiceInfo{externalPolicyLocal: true}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: true}, @@ -463,7 +463,7 @@ func TestCategorizeEndpoints(t *testing.T) { allEndpoints: sets.NewString("10.0.0.0:80", "10.0.0.1:80"), }, { name: "no cluster endpoints for iTP:Local internal-only service", - serviceInfo: &BaseServiceInfo{nodeLocalInternal: true}, + serviceInfo: &BaseServiceInfo{internalPolicyLocal: true}, endpoints: []Endpoint{ &BaseEndpointInfo{Endpoint: "10.0.0.0:80", Ready: true, IsLocal: false}, &BaseEndpointInfo{Endpoint: "10.0.0.1:80", Ready: true, IsLocal: true}, diff --git a/pkg/proxy/types.go b/pkg/proxy/types.go index a68d36109a8..7b16cd7b598 100644 --- a/pkg/proxy/types.go +++ b/pkg/proxy/types.go @@ -83,10 +83,10 @@ type ServicePort interface { HealthCheckNodePort() int // GetNodePort returns a service Node port if present. If return 0, it means not present. NodePort() int - // NodeLocalExternal returns if a service has only node local endpoints for external traffic. - NodeLocalExternal() bool - // NodeLocalInternal returns if a service has only node local endpoints for internal traffic. - NodeLocalInternal() bool + // ExternalPolicyLocal returns if a service has only node local endpoints for external traffic. + ExternalPolicyLocal() bool + // InternalPolicyLocal returns if a service has only node local endpoints for internal traffic. + InternalPolicyLocal() bool // InternalTrafficPolicy returns service InternalTrafficPolicy InternalTrafficPolicy() *v1.ServiceInternalTrafficPolicyType // HintsAnnotation returns the value of the v1.AnnotationTopologyAwareHints annotation.