diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 97089dc16d3..12fa88d2324 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -304,7 +304,7 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { podCIDR := "10.0.0.0/8" if ipt.IsIPv6() { ipfamily = v1.IPv6Protocol - podCIDR = "fd00::/64" + podCIDR = "fd00:10::/64" } detectLocal, _ := proxyutiliptables.NewDetectLocalByCIDR(podCIDR) @@ -2480,128 +2480,257 @@ func TestLoadBalancer(t *testing.T) { }) } -func TestNodePort(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "172.30.0.41" - svcPort := 80 - svcNodePort := 3001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, +// TestNodePorts tests NodePort services under various combinations of the +// --nodeport-addresses and --localhost-nodeports flags. +func TestNodePorts(t *testing.T) { + testCases := []struct { + name string + + family v1.IPFamily + localhostNodePorts bool + nodePortAddresses []string + + // allowAltNodeIP is true if we expect NodePort traffic on the alternate + // node IP to be accepted + allowAltNodeIP bool + + // expectFirewall is true if we expect KUBE-FIREWALL to be filled in with + // an anti-martian-packet rule + expectFirewall bool + }{ + { + name: "ipv4, localhost-nodeports enabled", + + family: v1.IPv4Protocol, + localhostNodePorts: true, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: true, + }, + { + name: "ipv4, localhost-nodeports disabled", + + family: v1.IPv4Protocol, + localhostNodePorts: false, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv4, localhost-nodeports disabled, localhost in nodeport-addresses", + + family: v1.IPv4Protocol, + localhostNodePorts: false, + nodePortAddresses: []string{"192.168.0.0/24", "127.0.0.1/32"}, + + allowAltNodeIP: false, + expectFirewall: false, + }, + { + name: "ipv4, localhost-nodeports enabled, multiple nodeport-addresses", + + family: v1.IPv4Protocol, + localhostNodePorts: false, + nodePortAddresses: []string{"192.168.0.0/24", "192.168.1.0/24", "2001:db8::/64"}, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv6, localhost-nodeports enabled", + + family: v1.IPv6Protocol, + localhostNodePorts: true, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv6, localhost-nodeports disabled", + + family: v1.IPv6Protocol, + localhostNodePorts: false, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv6, localhost-nodeports disabled, multiple nodeport-addresses", + + family: v1.IPv6Protocol, + localhostNodePorts: false, + nodePortAddresses: []string{"192.168.0.0/24", "192.168.1.0/24", "2001:db8::/64"}, + + allowAltNodeIP: false, + expectFirewall: false, + }, } - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ipt *iptablestest.FakeIPTables + var svcIP, epIP1, epIP2 string + if tc.family == v1.IPv4Protocol { + ipt = iptablestest.NewFake() + svcIP = "172.30.0.41" + epIP1 = "10.180.0.1" + epIP2 = "10.180.2.1" + } else { + ipt = iptablestest.NewIPv6Fake() + svcIP = "fd00:172:30::41" + epIP1 = "fd00:10:180::1" + epIP2 = "fd00:10:180::2:1" + } + fp := NewFakeProxier(ipt) + fp.localhostNodePorts = tc.localhostNodePorts + if tc.nodePortAddresses != nil { + fp.nodePortAddresses = proxyutil.NewNodePortAddresses(tc.family, tc.nodePortAddresses) + } - epIP := "10.180.0.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP}, - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) + makeServiceMap(fp, + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeNodePort + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: 3001, + }} + }), + ) - fp.syncProxyRules() + populateEndpointSlices(fp, + makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { + if tc.family == v1.IPv4Protocol { + eps.AddressType = discovery.AddressTypeIPv4 + } else { + eps.AddressType = discovery.AddressTypeIPv6 + } + eps.Endpoints = []discovery.Endpoint{{ + Addresses: []string{epIP1}, + NodeName: nil, + }, { + Addresses: []string{epIP2}, + NodeName: pointer.String(testHostname), + }} + eps.Ports = []discovery.EndpointPort{{ + Name: pointer.String("p80"), + Port: pointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), + ) - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) + fp.syncProxyRules() - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to cluster IP", - sourceIP: "10.0.0.2", - destIP: svcIP, - destPort: svcPort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: false, - }, - { - name: "external to nodePort", - sourceIP: testExternalClient, - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - { - name: "external to nodePort on secondary IP", - sourceIP: testExternalClient, - destIP: testNodeIPAlt, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - { - name: "node to nodePort", - sourceIP: testNodeIP, - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - { - name: "localhost to nodePort gets masqueraded", - sourceIP: "127.0.0.1", - destIP: "127.0.0.1", - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - }) + var podIP, externalClientIP, nodeIP, altNodeIP, localhostIP string + if tc.family == v1.IPv4Protocol { + podIP = "10.0.0.2" + externalClientIP = testExternalClient + nodeIP = testNodeIP + altNodeIP = testNodeIPAlt + localhostIP = "127.0.0.1" + } else { + podIP = "fd00:10::2" + externalClientIP = "2600:5200::1" + nodeIP = testNodeIPv6 + altNodeIP = testNodeIPv6Alt + localhostIP = "::1" + } + output := net.JoinHostPort(epIP1, "80") + ", " + net.JoinHostPort(epIP2, "80") + + // Basic tests are the same for all cases + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: podIP, + destIP: svcIP, + destPort: 80, + output: output, + masq: false, + }, + { + name: "external to nodePort", + sourceIP: externalClientIP, + destIP: nodeIP, + destPort: 3001, + output: output, + masq: true, + }, + { + name: "node to nodePort", + sourceIP: nodeIP, + destIP: nodeIP, + destPort: 3001, + output: output, + masq: true, + }, + }) + + // localhost to NodePort is only allowed in IPv4, and only if not disabled + if tc.family == v1.IPv4Protocol && tc.localhostNodePorts { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "localhost to nodePort gets masqueraded", + sourceIP: localhostIP, + destIP: localhostIP, + destPort: 3001, + output: output, + masq: true, + }, + }) + } else { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "localhost to nodePort is ignored", + sourceIP: localhostIP, + destIP: localhostIP, + destPort: 3001, + output: "", + }, + }) + } + + // NodePort on altNodeIP should be allowed, unless + // nodePortAddressess excludes altNodeIP + if tc.allowAltNodeIP { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "external to nodePort on secondary IP", + sourceIP: externalClientIP, + destIP: altNodeIP, + destPort: 3001, + output: output, + masq: true, + }, + }) + } else { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "secondary nodeIP ignores NodePorts", + sourceIP: externalClientIP, + destIP: altNodeIP, + destPort: 3001, + output: "", + }, + }) + } + + // We have to check the firewall rule manually rather than via + // runPacketFlowTests(), because the packet tracer doesn't + // implement conntrack states. + var expected string + if tc.expectFirewall { + expected = "-A KUBE-FIREWALL -m comment --comment \"block incoming localnet connections\" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP\n" + } + assertIPTablesChainEqual(t, getLine(), utiliptables.TableFilter, kubeletFirewallChain, expected, fp.iptablesData.String()) + }) + } } func TestHealthCheckNodePort(t *testing.T) { @@ -3093,474 +3222,6 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { }) } -func TestEnableLocalhostNodePortsIPv4(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = true - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-6KG6DFHVBKBK53RU - [0:0] - :KUBE-SEP-KDGX2M2ONE25PSWH - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.69.0.10 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -s 10.244.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.0.1:80 - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -s 10.244.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-6KG6DFHVBKBK53RU - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - COMMIT - `) - svcIP := "10.69.0.10" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "10.244.0.1" - epIP2 := "10.244.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestDisableLocalhostNodePortsIPv4(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = false - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-6KG6DFHVBKBK53RU - [0:0] - :KUBE-SEP-KDGX2M2ONE25PSWH - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.69.0.10 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL ! -d 127.0.0.0/8 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -s 10.244.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.0.1:80 - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -s 10.244.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-6KG6DFHVBKBK53RU - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - COMMIT - `) - svcIP := "10.69.0.10" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "10.244.0.1" - epIP2 := "10.244.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestDisableLocalhostNodePortsIPv4WithNodeAddress(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = false - fp.networkInterfacer.InterfaceAddrs() - fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"127.0.0.0/8"}) - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-6KG6DFHVBKBK53RU - [0:0] - :KUBE-SEP-KDGX2M2ONE25PSWH - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.69.0.10 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -s 10.244.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.0.1:80 - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -s 10.244.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-6KG6DFHVBKBK53RU - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - COMMIT - `) - svcIP := "10.69.0.10" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "10.244.0.1" - epIP2 := "10.244.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestEnableLocalhostNodePortsIPv6(t *testing.T) { - ipt := iptablestest.NewIPv6Fake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = true - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-LIGRYQQLSZN4UWQ5 - [0:0] - :KUBE-SEP-XJJ5QXWGJG344QDZ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d fd00:ab34::20 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL ! -d ::1/128 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -s ff06::c1 -j KUBE-MARK-MASQ - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c1]:80 - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -s ff06::c2 -j KUBE-MARK-MASQ - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c2]:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c1]:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-LIGRYQQLSZN4UWQ5 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - COMMIT - `) - svcIP := "fd00:ab34::20" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "ff06::c1" - epIP2 := "ff06::c2" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv6 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestDisableLocalhostNodePortsIPv6(t *testing.T) { - ipt := iptablestest.NewIPv6Fake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = false - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-LIGRYQQLSZN4UWQ5 - [0:0] - :KUBE-SEP-XJJ5QXWGJG344QDZ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d fd00:ab34::20 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL ! -d ::1/128 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -s ff06::c1 -j KUBE-MARK-MASQ - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c1]:80 - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -s ff06::c2 -j KUBE-MARK-MASQ - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c2]:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c1]:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-LIGRYQQLSZN4UWQ5 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - COMMIT - `) - svcIP := "fd00:ab34::20" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "ff06::c1" - epIP2 := "ff06::c2" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv6 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt)