From 43db55e93de797a5f2b3fe3af90a424c32d80f00 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 22 Sep 2023 10:42:25 -0400 Subject: [PATCH 01/13] Rename and extend TestOverallIPTablesRulesWithMultipleServices Rename TestOverallIPTablesRulesWithMultipleServices to just TestOverallIPTablesRules, and add one rule type we weren't previously testing (session affinity). --- pkg/proxy/iptables/proxier_test.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index ca00461124b..c0d62ff88cd 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1733,11 +1733,9 @@ func TestTracePackets(t *testing.T) { }) } -// TestOverallIPTablesRulesWithMultipleServices creates 4 types of services: ClusterIP, -// LoadBalancer, ExternalIP and NodePort and verifies if the NAT table rules created -// are exactly the same as what is expected. This test provides an overall view of how -// the NAT table rules look like with the different jumps. -func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { +// TestOverallIPTablesRules creates a variety of services and verifies that the generated +// rules are exactly as expected. +func TestOverallIPTablesRules(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) metrics.RegisterMetrics() @@ -1792,7 +1790,8 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { TargetPort: intstr.FromInt32(80), }} }), - // create LoadBalancer service with Cluster traffic policy and source ranges + // create LoadBalancer service with Cluster traffic policy, source ranges, + // and session affinity makeTestService("ns5", "svc5", func(svc *v1.Service) { svc.Spec.Type = "LoadBalancer" svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyCluster @@ -1810,6 +1809,12 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { // Extra whitespace to ensure that invalid value will not result // in a crash, for backward compatibility. svc.Spec.LoadBalancerSourceRanges = []string{" 203.0.113.0/25"} + + svcSessionAffinityTimeout := int32(10800) + svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP + svc.Spec.SessionAffinityConfig = &v1.SessionAffinityConfig{ + ClientIP: &v1.ClientIPConfig{TimeoutSeconds: &svcSessionAffinityTimeout}, + } }), // create ClusterIP service with no endpoints makeTestService("ns6", "svc6", func(svc *v1.Service) { @@ -1963,7 +1968,7 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 -A KUBE-SEP-I77PXRDZVX7PMWMN -m comment --comment ns5/svc5:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ - -A KUBE-SEP-I77PXRDZVX7PMWMN -m comment --comment ns5/svc5:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 + -A KUBE-SEP-I77PXRDZVX7PMWMN -m comment --comment ns5/svc5:p80 -m recent --name KUBE-SEP-I77PXRDZVX7PMWMN --set -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ @@ -1978,6 +1983,7 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 -> 10.180.0.2:80" -j KUBE-SEP-RS4RBKLTHTF2IUXJ -A KUBE-SVC-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 cluster IP" -m tcp -p tcp -d 172.30.0.45 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 -> 10.180.0.3:80" -m recent --name KUBE-SEP-I77PXRDZVX7PMWMN --rcheck --seconds 10800 --reap -j KUBE-SEP-I77PXRDZVX7PMWMN -A KUBE-SVC-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 -> 10.180.0.3:80" -j KUBE-SEP-I77PXRDZVX7PMWMN -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 -> 10.180.0.3:80" -j KUBE-SEP-OYPFS5VJICHGATKP From d57a51d0a901fa5df9b0f43628a2d4d5845fc255 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 7 Jul 2023 13:55:52 -0400 Subject: [PATCH 02/13] Remove assertIPTablesRulesEqual from InternalTrafficPolicy test Just use the flow tests. Also, add a new test for a missing case. --- pkg/proxy/iptables/proxier_test.go | 145 ++++++----------------------- 1 file changed, 27 insertions(+), 118 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index c0d62ff88cd..8e3b97fd80d 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -5466,7 +5466,7 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) { // This test ensures that the iptables proxier supports translating Endpoints to // iptables output when internalTrafficPolicy is specified -func TestInternalTrafficPolicyE2E(t *testing.T) { +func TestInternalTrafficPolicy(t *testing.T) { type endpoint struct { ip string hostname string @@ -5475,55 +5475,12 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { cluster := v1.ServiceInternalTrafficPolicyCluster local := v1.ServiceInternalTrafficPolicyLocal - clusterExpectedIPTables := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] - :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] - :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT - COMMIT - `) - testCases := []struct { - name string - line int - internalTrafficPolicy *v1.ServiceInternalTrafficPolicy - endpoints []endpoint - expectEndpointRule bool - expectedIPTablesWithSlice string - flowTests []packetFlowTest + name string + line int + internalTrafficPolicy *v1.ServiceInternalTrafficPolicy + endpoints []endpoint + flowTests []packetFlowTest }{ { name: "internalTrafficPolicy is cluster", @@ -5534,8 +5491,6 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { {"10.0.1.2", "host1"}, {"10.0.1.3", "host2"}, }, - expectEndpointRule: true, - expectedIPTablesWithSlice: clusterExpectedIPTables, flowTests: []packetFlowTest{ { name: "pod to ClusterIP hits all endpoints", @@ -5548,7 +5503,7 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { }, }, { - name: "internalTrafficPolicy is local and there are local endpoints", + name: "internalTrafficPolicy is local and there is one local endpoint", line: getLine(), internalTrafficPolicy: &local, endpoints: []endpoint{ @@ -5556,39 +5511,6 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { {"10.0.1.2", "host1"}, {"10.0.1.3", "host2"}, }, - expectEndpointRule: true, - expectedIPTablesWithSlice: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] - :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -j KUBE-SEP-3JOIVZTXZZRGORX4 - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to ClusterIP hits only local endpoint", @@ -5600,6 +5522,26 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { }, }, }, + { + name: "internalTrafficPolicy is local and there are multiple local endpoints", + line: getLine(), + internalTrafficPolicy: &local, + endpoints: []endpoint{ + {"10.0.1.1", testHostname}, + {"10.0.1.2", testHostname}, + {"10.0.1.3", "host2"}, + }, + flowTests: []packetFlowTest{ + { + name: "pod to ClusterIP hits all local endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80", + masq: false, + }, + }, + }, { name: "internalTrafficPolicy is local and there are no local endpoints", line: getLine(), @@ -5609,33 +5551,6 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { {"10.0.1.2", "host1"}, {"10.0.1.3", "host2"}, }, - expectEndpointRule: false, - expectedIPTablesWithSlice: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 has no local endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j DROP - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `), flowTests: []packetFlowTest{ { name: "no endpoints", @@ -5695,16 +5610,10 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - assertIPTablesRulesEqual(t, tc.line, true, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) runPacketFlowTests(t, tc.line, ipt, testNodeIP, tc.flowTests) fp.OnEndpointSliceDelete(endpointSlice) fp.syncProxyRules() - if tc.expectEndpointRule { - fp.OnEndpointSliceDelete(endpointSlice) - fp.syncProxyRules() - assertIPTablesRulesNotEqual(t, tc.line, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) - } runPacketFlowTests(t, tc.line, ipt, testNodeIP, []packetFlowTest{ { name: "endpoints deleted", From 4438f5e436b1414b4cc2fef672328e131e544e82 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 13 Jul 2023 10:30:13 -0400 Subject: [PATCH 03/13] Remove assertIPTablesRulesEqual checks from terminating endpoints tests The flow tests sufficiently check the results. Also remove some irrelevant bits of the Service definition that don't affect these tests. --- pkg/proxy/iptables/proxier_test.go | 384 +---------------------------- 1 file changed, 8 insertions(+), 376 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 8e3b97fd80d..5af5159d3b2 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -5630,14 +5630,12 @@ func TestInternalTrafficPolicy(t *testing.T) { // TestTerminatingEndpointsTrafficPolicyLocal tests that when there are local ready and // ready + terminating endpoints, only the ready endpoints are used. func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { - timeout := v1.DefaultClientIPServiceAffinitySeconds service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"}, Spec: v1.ServiceSpec{ ClusterIP: "172.30.1.1", Type: v1.ServiceTypeLoadBalancer, ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, - Selector: map[string]string{"foo": "bar"}, Ports: []v1.ServicePort{ { Name: "", @@ -5647,12 +5645,6 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { }, }, HealthCheckNodePort: 30000, - SessionAffinity: v1.ServiceAffinityClientIP, - SessionAffinityConfig: &v1.SessionAffinityConfig{ - ClientIP: &v1.ClientIPConfig{ - TimeoutSeconds: &timeout, - }, - }, }, Status: v1.ServiceStatus{ LoadBalancer: v1.LoadBalancerStatus{ @@ -5664,12 +5656,10 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { } testcases := []struct { - name string - line int - endpointslice *discovery.EndpointSlice - expectedIPTables string - noUsableEndpoints bool - flowTests []packetFlowTest + name string + line int + endpointslice *discovery.EndpointSlice + flowTests []packetFlowTest }{ { name: "ready endpoints exist", @@ -5737,61 +5727,6 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { }, }, }, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] - :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] - :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -j KUBE-SEP-IO5XOSKPAXIFQXAJ - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP", @@ -5869,57 +5804,6 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { }, }, }, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] - :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] - :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP", @@ -5968,46 +5852,6 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { }, }, }, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no local endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP", @@ -6063,35 +5907,6 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { }, }, }, - noUsableEndpoints: true, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP, no usable endpoints", @@ -6122,17 +5937,10 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() - assertIPTablesRulesEqual(t, testcase.line, true, testcase.expectedIPTables, fp.iptablesData.String()) runPacketFlowTests(t, testcase.line, ipt, testNodeIP, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() - if testcase.noUsableEndpoints { - // Deleting the EndpointSlice should have had no effect - assertIPTablesRulesEqual(t, testcase.line, true, testcase.expectedIPTables, fp.iptablesData.String()) - } else { - assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) - } runPacketFlowTests(t, testcase.line, ipt, testNodeIP, []packetFlowTest{ { name: "pod to clusterIP after endpoints deleted", @@ -6156,14 +5964,12 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { // TestTerminatingEndpointsTrafficPolicyCluster tests that when there are cluster-wide // ready and ready + terminating endpoints, only the ready endpoints are used. func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { - timeout := v1.DefaultClientIPServiceAffinitySeconds service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"}, Spec: v1.ServiceSpec{ ClusterIP: "172.30.1.1", Type: v1.ServiceTypeLoadBalancer, ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyCluster, - Selector: map[string]string{"foo": "bar"}, Ports: []v1.ServicePort{ { Name: "", @@ -6173,12 +5979,6 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { }, }, HealthCheckNodePort: 30000, - SessionAffinity: v1.ServiceAffinityClientIP, - SessionAffinityConfig: &v1.SessionAffinityConfig{ - ClientIP: &v1.ClientIPConfig{ - TimeoutSeconds: &timeout, - }, - }, }, Status: v1.ServiceStatus{ LoadBalancer: v1.LoadBalancerStatus{ @@ -6190,12 +5990,10 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { } testcases := []struct { - name string - line int - endpointslice *discovery.EndpointSlice - expectedIPTables string - noUsableEndpoints bool - flowTests []packetFlowTest + name string + line int + endpointslice *discovery.EndpointSlice + flowTests []packetFlowTest }{ { name: "ready endpoints exist", @@ -6262,53 +6060,6 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { }, }, }, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] - :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] - :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP", @@ -6386,53 +6137,6 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { }, }, }, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] - :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] - :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-XGJFVO3L2O5SRFNT - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP", @@ -6479,43 +6183,6 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { }, }, }, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5 -j KUBE-MARK-MASQ - -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP", @@ -6573,34 +6240,6 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { }, }, }, - noUsableEndpoints: true, - expectedIPTables: dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j REJECT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `), flowTests: []packetFlowTest{ { name: "pod to clusterIP", @@ -6632,17 +6271,10 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() - assertIPTablesRulesEqual(t, testcase.line, true, testcase.expectedIPTables, fp.iptablesData.String()) runPacketFlowTests(t, testcase.line, ipt, testNodeIP, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() - if testcase.noUsableEndpoints { - // Deleting the EndpointSlice should have had no effect - assertIPTablesRulesEqual(t, testcase.line, true, testcase.expectedIPTables, fp.iptablesData.String()) - } else { - assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) - } runPacketFlowTests(t, testcase.line, ipt, testNodeIP, []packetFlowTest{ { name: "pod to clusterIP after endpoints deleted", From 0ab0e404b83862819b4507b83ff654b9dded1ca7 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 22 Sep 2023 11:12:33 -0400 Subject: [PATCH 04/13] Drop the now-unused assertIPTablesRulesNotEqual Previously this was used to assert "something changed since the last sync", but we already have packet flow tests in all of those cases now to assert that the *specific* something we care about changed. --- pkg/proxy/iptables/proxier_test.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 5af5159d3b2..7e3d1519818 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1312,34 +1312,6 @@ func assertIPTablesRulesEqual(t *testing.T, line int, checkConsistency bool, exp } } -// assertIPTablesRulesNotEqual asserts that the generated rules in result DON'T match the -// rules in expected, ignoring irrelevant ordering differences. -func assertIPTablesRulesNotEqual(t *testing.T, line int, expected, result string) { - expected = strings.TrimLeft(expected, " \t\n") - - result, err := sortIPTablesRules(strings.TrimLeft(result, " \t\n")) - if err != nil { - t.Fatalf("%s", err) - } - - lineStr := "" - if line != 0 { - lineStr = fmt.Sprintf(" (from line %d)", line) - } - if cmp.Equal(expected, result) { - t.Errorf("rules do not differ%s:\nfull result:\n```\n%s```", lineStr, result) - } - - err = checkIPTablesRuleJumps(expected) - if err != nil { - t.Fatalf("%s", err) - } - err = checkIPTablesRuleJumps(result) - if err != nil { - t.Fatalf("%s", err) - } -} - // addressMatches helps test whether an iptables rule such as "! -s 192.168.0.0/16" matches // ipStr. address.Value is either an IP address ("1.2.3.4") or a CIDR string // ("1.2.3.0/24"). From a25fb03c00a14d11b4ddc5282b2e23056f9c34e6 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 22 Sep 2023 11:37:26 -0400 Subject: [PATCH 05/13] Add assertIPTablesChainEquals, to streamline a few tests Rather than checking the entire iptables dump, only check a single chain. --- pkg/proxy/iptables/proxier_test.go | 121 +++++++++++++---------------- 1 file changed, 54 insertions(+), 67 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 7e3d1519818..9c088da13ff 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1312,6 +1312,32 @@ func assertIPTablesRulesEqual(t *testing.T, line int, checkConsistency bool, exp } } +// assertIPTablesChainEqual asserts that the indicated chain in the indicated table in +// result contains exactly the rules in expected (in that order). +func assertIPTablesChainEqual(t *testing.T, line int, table utiliptables.Table, chain utiliptables.Chain, expected, result string) { + expected = strings.TrimLeft(expected, " \t\n") + + dump, err := iptablestest.ParseIPTablesDump(strings.TrimLeft(result, " \t\n")) + if err != nil { + t.Fatalf("%s", err) + } + + result = "" + if ch, _ := dump.GetChain(table, chain); ch != nil { + for _, rule := range ch.Rules { + result += rule.Raw + "\n" + } + } + + lineStr := "" + if line != 0 { + lineStr = fmt.Sprintf(" (from line %d)", line) + } + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("rules do not match%s:\ndiff:\n%s\nfull result:\n```\n%s```", lineStr, diff, result) + } +} + // addressMatches helps test whether an iptables rule such as "! -s 192.168.0.0/16" matches // ipStr. address.Value is either an IP address ("1.2.3.4") or a CIDR string // ("1.2.3.0/24"). @@ -2544,86 +2570,47 @@ func TestHealthCheckNodePort(t *testing.T) { } func TestDropInvalidRule(t *testing.T) { - for _, testcase := range []bool{false, true} { - t.Run(fmt.Sprintf("tcpLiberal %t", testcase), func(t *testing.T) { + for _, tcpLiberal := range []bool{false, true} { + t.Run(fmt.Sprintf("tcpLiberal %t", tcpLiberal), func(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) - fp.conntrackTCPLiberal = testcase + fp.conntrackTCPLiberal = tcpLiberal fp.syncProxyRules() - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP`) - if !testcase { - expected += "\n-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP" + var expected string + if !tcpLiberal { + expected = "-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP" } - expected += dedent.Dedent(` - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `) + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) + assertIPTablesChainEqual(t, getLine(), utiliptables.TableFilter, kubeForwardChain, expected, fp.iptablesData.String()) }) } } func TestMasqueradeRule(t *testing.T) { - for _, testcase := range []bool{false, true} { - ipt := iptablestest.NewFake().SetHasRandomFully(testcase) - fp := NewFakeProxier(ipt) - fp.syncProxyRules() + for _, randomFully := range []bool{false, true} { + t.Run(fmt.Sprintf("randomFully %t", randomFully), func(t *testing.T) { + ipt := iptablestest.NewFake().SetHasRandomFully(randomFully) + fp := NewFakeProxier(ipt) + fp.syncProxyRules() - expectedFmt := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE%s - COMMIT - `) - var expected string - if testcase { - expected = fmt.Sprintf(expectedFmt, " --random-fully") - } else { - expected = fmt.Sprintf(expectedFmt, "") - } - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) + expectedFmt := dedent.Dedent(` + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE%s + `) + var expected string + if randomFully { + expected = fmt.Sprintf(expectedFmt, " --random-fully") + } else { + expected = fmt.Sprintf(expectedFmt, "") + } + assertIPTablesChainEqual(t, getLine(), utiliptables.TableNAT, kubePostroutingChain, expected, fp.iptablesData.String()) + }) } } From 0910fe4b983e078dbbfe38aa12e23f9e6042be37 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sun, 25 Jun 2023 15:20:16 -0400 Subject: [PATCH 06/13] Extend iptables packet tracer to check the protocol --- pkg/proxy/iptables/proxier_test.go | 47 ++++++++++++++++++------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 9c088da13ff..d705c74a22d 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1395,9 +1395,8 @@ func newIPTablesTracer(t *testing.T, ipt *iptablestest.FakeIPTables, nodeIP stri } // ruleMatches checks if the given iptables rule matches (at least probabilistically) a -// packet with the given sourceIP, destIP, and destPort. (Note that protocol is currently -// ignored.) -func (tracer *iptablesTracer) ruleMatches(rule *iptablestest.Rule, sourceIP, destIP, destPort string) bool { +// packet with the given sourceIP, destIP, and destPort. +func (tracer *iptablesTracer) ruleMatches(rule *iptablestest.Rule, sourceIP, protocol, destIP, destPort string) bool { // The sub-rules within an iptables rule are ANDed together, so the rule only // matches if all of them match. So go through the subrules, and if any of them // DON'T match, then fail. @@ -1415,6 +1414,10 @@ func (tracer *iptablesTracer) ruleMatches(rule *iptablestest.Rule, sourceIP, des } } + if rule.Protocol != nil && !rule.Protocol.Matches(protocol) { + return false + } + if rule.DestinationAddress != nil && !addressMatches(tracer.t, rule.DestinationAddress, destIP) { return false } @@ -1442,7 +1445,7 @@ func (tracer *iptablesTracer) ruleMatches(rule *iptablestest.Rule, sourceIP, des // runChain runs the given packet through the rules in the given table and chain, updating // tracer's internal state accordingly. It returns true if it hits a terminal action. -func (tracer *iptablesTracer) runChain(table utiliptables.Table, chain utiliptables.Chain, sourceIP, destIP, destPort string) bool { +func (tracer *iptablesTracer) runChain(table utiliptables.Table, chain utiliptables.Chain, sourceIP, protocol, destIP, destPort string) bool { c, _ := tracer.ipt.Dump.GetChain(table, chain) if c == nil { return false @@ -1453,7 +1456,7 @@ func (tracer *iptablesTracer) runChain(table utiliptables.Table, chain utiliptab continue } - if !tracer.ruleMatches(rule, sourceIP, destIP, destPort) { + if !tracer.ruleMatches(rule, sourceIP, protocol, destIP, destPort) { continue } // record the matched rule for debugging purposes @@ -1476,7 +1479,7 @@ func (tracer *iptablesTracer) runChain(table utiliptables.Table, chain utiliptab default: // We got a "-j KUBE-SOMETHING", so process that chain - terminated := tracer.runChain(table, utiliptables.Chain(rule.Jump.Value), sourceIP, destIP, destPort) + terminated := tracer.runChain(table, utiliptables.Chain(rule.Jump.Value), sourceIP, protocol, destIP, destPort) // If the subchain hit a terminal rule AND the rule that sent us // to that chain was non-probabilistic, then this chain terminates @@ -1492,18 +1495,19 @@ func (tracer *iptablesTracer) runChain(table utiliptables.Table, chain utiliptab return false } -// tracePacket determines what would happen to a packet with the given sourceIP, destIP, -// and destPort, given the indicated iptables ruleData. nodeIP is the local node IP (for -// rules matching "LOCAL"). +// tracePacket determines what would happen to a packet with the given sourceIP, protocol, +// destIP, and destPort, given the indicated iptables ruleData. nodeIP is the local node +// IP (for rules matching "LOCAL"). (The protocol value should be lowercase as in iptables +// rules, not uppercase as in corev1.) // // The return values are: an array of matched rules (for debugging), the final packet // destinations (a comma-separated list of IPs, or one of the special targets "ACCEPT", // "DROP", or "REJECT"), and whether the packet would be masqueraded. -func tracePacket(t *testing.T, ipt *iptablestest.FakeIPTables, sourceIP, destIP, destPort, nodeIP string) ([]string, string, bool) { +func tracePacket(t *testing.T, ipt *iptablestest.FakeIPTables, sourceIP, protocol, destIP, destPort, nodeIP string) ([]string, string, bool) { tracer := newIPTablesTracer(t, ipt, nodeIP) // nat:PREROUTING goes first - tracer.runChain(utiliptables.TableNAT, utiliptables.ChainPrerouting, sourceIP, destIP, destPort) + tracer.runChain(utiliptables.TableNAT, utiliptables.ChainPrerouting, sourceIP, protocol, destIP, destPort) // After the PREROUTING rules run, pending DNATs are processed (which would affect // the destination IP that later rules match against). @@ -1515,10 +1519,10 @@ func tracePacket(t *testing.T, ipt *iptablestest.FakeIPTables, sourceIP, destIP, // inbound, outbound, or intra-host packet, which we don't know. So we just run // the interesting tables manually. (Theoretically this could cause conflicts in // the future in which case we'd have to do something more complicated.) - tracer.runChain(utiliptables.TableFilter, kubeServicesChain, sourceIP, destIP, destPort) - tracer.runChain(utiliptables.TableFilter, kubeExternalServicesChain, sourceIP, destIP, destPort) - tracer.runChain(utiliptables.TableFilter, kubeNodePortsChain, sourceIP, destIP, destPort) - tracer.runChain(utiliptables.TableFilter, kubeProxyFirewallChain, sourceIP, destIP, destPort) + tracer.runChain(utiliptables.TableFilter, kubeServicesChain, sourceIP, protocol, destIP, destPort) + tracer.runChain(utiliptables.TableFilter, kubeExternalServicesChain, sourceIP, protocol, destIP, destPort) + tracer.runChain(utiliptables.TableFilter, kubeNodePortsChain, sourceIP, protocol, destIP, destPort) + tracer.runChain(utiliptables.TableFilter, kubeProxyFirewallChain, sourceIP, protocol, destIP, destPort) // Finally, the nat:POSTROUTING rules run, but the only interesting thing that // happens there is that the masquerade mark gets turned into actual masquerading. @@ -1529,6 +1533,7 @@ func tracePacket(t *testing.T, ipt *iptablestest.FakeIPTables, sourceIP, destIP, type packetFlowTest struct { name string sourceIP string + protocol v1.Protocol destIP string destPort int output string @@ -1542,7 +1547,11 @@ func runPacketFlowTests(t *testing.T, line int, ipt *iptablestest.FakeIPTables, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - matches, output, masq := tracePacket(t, ipt, tc.sourceIP, tc.destIP, fmt.Sprintf("%d", tc.destPort), nodeIP) + protocol := strings.ToLower(string(tc.protocol)) + if protocol == "" { + protocol = "tcp" + } + matches, output, masq := tracePacket(t, ipt, tc.sourceIP, protocol, tc.destIP, fmt.Sprintf("%d", tc.destPort), nodeIP) var errors []string if output != tc.output { errors = append(errors, fmt.Sprintf("wrong output: expected %q got %q", tc.output, output)) @@ -1551,8 +1560,8 @@ func runPacketFlowTests(t *testing.T, line int, ipt *iptablestest.FakeIPTables, errors = append(errors, fmt.Sprintf("wrong masq: expected %v got %v", tc.masq, masq)) } if errors != nil { - t.Errorf("Test %q of a packet from %s to %s:%d%s got result:\n%s\n\nBy matching:\n%s\n\n", - tc.name, tc.sourceIP, tc.destIP, tc.destPort, lineStr, strings.Join(errors, "\n"), strings.Join(matches, "\n")) + t.Errorf("Test %q of a %s packet from %s to %s:%d%s got result:\n%s\n\nBy matching:\n%s\n\n", + tc.name, protocol, tc.sourceIP, tc.destIP, tc.destPort, lineStr, strings.Join(errors, "\n"), strings.Join(matches, "\n")) } }) } @@ -2139,6 +2148,7 @@ func TestClusterIPEndpointsMore(t *testing.T) { { name: "cluster IP accepted", sourceIP: "10.180.0.2", + protocol: v1.ProtocolSCTP, destIP: "172.30.0.41", destPort: 80, output: "10.180.0.1:80", @@ -2147,6 +2157,7 @@ func TestClusterIPEndpointsMore(t *testing.T) { { name: "hairpin to cluster IP", sourceIP: "10.180.0.1", + protocol: v1.ProtocolSCTP, destIP: "172.30.0.41", destPort: 80, output: "10.180.0.1:80", From ce7ffa81754e06cc136f7c5bf750211cf1459ff6 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 22 Sep 2023 09:53:29 -0400 Subject: [PATCH 07/13] Extend iptables packet tracer to support multiple node IPs --- pkg/proxy/iptables/proxier_test.go | 100 +++++++++++++++++------------ 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index d705c74a22d..f13dd511edd 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -288,9 +288,15 @@ func TestDeleteEndpointConnections(t *testing.T) { const testHostname = "test-hostname" const testNodeIP = "192.168.0.2" +const testNodeIPAlt = "192.168.1.2" +const testExternalIP = "192.168.99.11" +const testNodeIPv6 = "2001:db8::1" +const testNodeIPv6Alt = "2001:db8:1::2" const testExternalClient = "203.0.113.2" const testExternalClientBlocked = "203.0.113.130" +var testNodeIPs = []string{testNodeIP, testNodeIPAlt, testExternalIP, testNodeIPv6, testNodeIPv6Alt} + func NewFakeProxier(ipt utiliptables.Interface) *Proxier { // TODO: Call NewProxier after refactoring out the goroutine // invocation into a Run() method. @@ -312,9 +318,10 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} addrs1 := []net.Addr{ &net.IPNet{IP: netutils.ParseIPSloppy(testNodeIP), Mask: net.CIDRMask(24, 32)}, - // (This IP never actually gets used; it's only here to test that it gets - // filtered out correctly in the IPv4 nodeport tests.) - &net.IPNet{IP: netutils.ParseIPSloppy("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, + &net.IPNet{IP: netutils.ParseIPSloppy(testNodeIPAlt), Mask: net.CIDRMask(24, 32)}, + &net.IPNet{IP: netutils.ParseIPSloppy(testExternalIP), Mask: net.CIDRMask(24, 32)}, + &net.IPNet{IP: netutils.ParseIPSloppy(testNodeIPv6), Mask: net.CIDRMask(64, 128)}, + &net.IPNet{IP: netutils.ParseIPSloppy(testNodeIPv6Alt), Mask: net.CIDRMask(64, 128)}, } networkInterfacer.AddInterfaceAddr(&itf1, addrs1) @@ -1367,9 +1374,9 @@ func addressMatches(t *testing.T, address *iptablestest.IPTablesValue, ipStr str // iptablesTracer holds data used while virtually tracing a packet through a set of // iptables rules type iptablesTracer struct { - ipt *iptablestest.FakeIPTables - nodeIP string - t *testing.T + ipt *iptablestest.FakeIPTables + localIPs sets.Set[string] + t *testing.T // matches accumulates the list of rules that were matched, for debugging purposes. matches []string @@ -1383,14 +1390,17 @@ type iptablesTracer struct { markMasq bool } -// newIPTablesTracer creates an iptablesTracer. nodeIP is the IP to treat as the local -// node IP (for determining whether rules with "--src-type LOCAL" or "--dst-type LOCAL" +// newIPTablesTracer creates an iptablesTracer. nodeIPs are the IPs to treat as local +// node IPs (for determining whether rules with "--src-type LOCAL" or "--dst-type LOCAL" // match). -func newIPTablesTracer(t *testing.T, ipt *iptablestest.FakeIPTables, nodeIP string) *iptablesTracer { +func newIPTablesTracer(t *testing.T, ipt *iptablestest.FakeIPTables, nodeIPs []string) *iptablesTracer { + localIPs := sets.New("127.0.0.1", "::1") + localIPs.Insert(nodeIPs...) + return &iptablesTracer{ - ipt: ipt, - nodeIP: nodeIP, - t: t, + ipt: ipt, + localIPs: localIPs, + t: t, } } @@ -1406,7 +1416,7 @@ func (tracer *iptablesTracer) ruleMatches(rule *iptablestest.Rule, sourceIP, pro } if rule.SourceType != nil { addrtype := "not-matched" - if sourceIP == tracer.nodeIP || sourceIP == "127.0.0.1" { + if tracer.localIPs.Has(sourceIP) { addrtype = "LOCAL" } if !rule.SourceType.Matches(addrtype) { @@ -1423,7 +1433,7 @@ func (tracer *iptablesTracer) ruleMatches(rule *iptablestest.Rule, sourceIP, pro } if rule.DestinationType != nil { addrtype := "not-matched" - if destIP == tracer.nodeIP || destIP == "127.0.0.1" { + if tracer.localIPs.Has(destIP) { addrtype = "LOCAL" } if !rule.DestinationType.Matches(addrtype) { @@ -1503,8 +1513,8 @@ func (tracer *iptablesTracer) runChain(table utiliptables.Table, chain utiliptab // The return values are: an array of matched rules (for debugging), the final packet // destinations (a comma-separated list of IPs, or one of the special targets "ACCEPT", // "DROP", or "REJECT"), and whether the packet would be masqueraded. -func tracePacket(t *testing.T, ipt *iptablestest.FakeIPTables, sourceIP, protocol, destIP, destPort, nodeIP string) ([]string, string, bool) { - tracer := newIPTablesTracer(t, ipt, nodeIP) +func tracePacket(t *testing.T, ipt *iptablestest.FakeIPTables, sourceIP, protocol, destIP, destPort string, nodeIPs []string) ([]string, string, bool) { + tracer := newIPTablesTracer(t, ipt, nodeIPs) // nat:PREROUTING goes first tracer.runChain(utiliptables.TableNAT, utiliptables.ChainPrerouting, sourceIP, protocol, destIP, destPort) @@ -1540,7 +1550,7 @@ type packetFlowTest struct { masq bool } -func runPacketFlowTests(t *testing.T, line int, ipt *iptablestest.FakeIPTables, nodeIP string, testCases []packetFlowTest) { +func runPacketFlowTests(t *testing.T, line int, ipt *iptablestest.FakeIPTables, nodeIPs []string, testCases []packetFlowTest) { lineStr := "" if line != 0 { lineStr = fmt.Sprintf(" (from line %d)", line) @@ -1551,7 +1561,7 @@ func runPacketFlowTests(t *testing.T, line int, ipt *iptablestest.FakeIPTables, if protocol == "" { protocol = "tcp" } - matches, output, masq := tracePacket(t, ipt, tc.sourceIP, protocol, tc.destIP, fmt.Sprintf("%d", tc.destPort), nodeIP) + matches, output, masq := tracePacket(t, ipt, tc.sourceIP, protocol, tc.destIP, fmt.Sprintf("%d", tc.destPort), nodeIPs) var errors []string if output != tc.output { errors = append(errors, fmt.Sprintf("wrong output: expected %q got %q", tc.output, output)) @@ -1686,7 +1696,7 @@ func TestTracePackets(t *testing.T) { t.Fatalf("Restore of test data failed: %v", err) } - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "no match", sourceIP: "10.0.0.2", @@ -2060,7 +2070,7 @@ func TestClusterIPReject(t *testing.T) { assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "cluster IP rejected", sourceIP: "10.0.0.2", @@ -2144,7 +2154,7 @@ func TestClusterIPEndpointsMore(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "cluster IP accepted", sourceIP: "10.180.0.2", @@ -2269,7 +2279,7 @@ func TestLoadBalancer(t *testing.T) { assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP", sourceIP: "10.0.0.2", @@ -2458,7 +2468,7 @@ func TestNodePort(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP", sourceIP: "10.0.0.2", @@ -2475,6 +2485,14 @@ func TestNodePort(t *testing.T) { output: fmt.Sprintf("%s:%d", epIP, svcPort), masq: true, }, + { + name: "external to nodePort on secondary IP", + sourceIP: testExternalClient, + destIP: testNodeIPAlt, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, { name: "node to nodePort", sourceIP: testNodeIP, @@ -2555,7 +2573,7 @@ func TestHealthCheckNodePort(t *testing.T) { assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "firewall accepts HealthCheckNodePort", sourceIP: "1.2.3.4", @@ -2569,7 +2587,7 @@ func TestHealthCheckNodePort(t *testing.T) { fp.OnServiceDelete(svc) fp.syncProxyRules() - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "HealthCheckNodePort no longer has any rule", sourceIP: "1.2.3.4", @@ -2681,7 +2699,7 @@ func TestExternalIPsReject(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "cluster IP with no endpoints", sourceIP: "10.0.0.2", @@ -2791,7 +2809,7 @@ func TestOnlyLocalExternalIPs(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "cluster IP hits both endpoints", sourceIP: "10.0.0.2", @@ -2900,7 +2918,7 @@ func TestNonLocalExternalIPs(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP", sourceIP: "10.0.0.2", @@ -2975,7 +2993,7 @@ func TestNodePortReject(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP", sourceIP: "10.0.0.2", @@ -3069,7 +3087,7 @@ func TestLoadBalancerReject(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP", sourceIP: "10.0.0.2", @@ -3203,7 +3221,7 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP hits both endpoints", sourceIP: "10.0.0.2", @@ -3848,7 +3866,7 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable assertIPTablesRulesEqual(t, line, true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, line, ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, line, ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP hit both endpoints", sourceIP: "10.0.0.2", @@ -3876,7 +3894,7 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable if fp.localDetector.IsImplemented() { // pod-to-NodePort is treated as internal traffic, so we see both endpoints - runPacketFlowTests(t, line, ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, line, ipt, testNodeIPs, []packetFlowTest{ { name: "pod to NodePort hits both endpoints", sourceIP: "10.0.0.2", @@ -3889,7 +3907,7 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable } else { // pod-to-NodePort is (incorrectly) treated as external traffic // when there is no LocalTrafficDetector. - runPacketFlowTests(t, line, ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, line, ipt, testNodeIPs, []packetFlowTest{ { name: "pod to NodePort hits only local endpoint", sourceIP: "10.0.0.2", @@ -5580,11 +5598,11 @@ func TestInternalTrafficPolicy(t *testing.T) { fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - runPacketFlowTests(t, tc.line, ipt, testNodeIP, tc.flowTests) + runPacketFlowTests(t, tc.line, ipt, testNodeIPs, tc.flowTests) fp.OnEndpointSliceDelete(endpointSlice) fp.syncProxyRules() - runPacketFlowTests(t, tc.line, ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, tc.line, ipt, testNodeIPs, []packetFlowTest{ { name: "endpoints deleted", sourceIP: "10.0.0.2", @@ -5907,11 +5925,11 @@ func TestTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() - runPacketFlowTests(t, testcase.line, ipt, testNodeIP, testcase.flowTests) + runPacketFlowTests(t, testcase.line, ipt, testNodeIPs, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() - runPacketFlowTests(t, testcase.line, ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, testcase.line, ipt, testNodeIPs, []packetFlowTest{ { name: "pod to clusterIP after endpoints deleted", sourceIP: "10.0.0.2", @@ -6241,11 +6259,11 @@ func TestTerminatingEndpointsTrafficPolicyCluster(t *testing.T) { fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() - runPacketFlowTests(t, testcase.line, ipt, testNodeIP, testcase.flowTests) + runPacketFlowTests(t, testcase.line, ipt, testNodeIPs, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() - runPacketFlowTests(t, testcase.line, ipt, testNodeIP, []packetFlowTest{ + runPacketFlowTests(t, testcase.line, ipt, testNodeIPs, []packetFlowTest{ { name: "pod to clusterIP after endpoints deleted", sourceIP: "10.0.0.2", @@ -6832,7 +6850,7 @@ func TestInternalExternalMasquerade(t *testing.T) { if overridesApplied != len(tc.overrides) { t.Errorf("%d overrides did not match any test case name!", len(tc.overrides)-overridesApplied) } - runPacketFlowTests(t, tc.line, ipt, testNodeIP, tcFlowTests) + runPacketFlowTests(t, tc.line, ipt, testNodeIPs, tcFlowTests) }) } } From 2435da11d5d8968d745d3a456aa12f2ebce11af8 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 7 Jul 2023 14:47:24 -0400 Subject: [PATCH 08/13] Rewrite TestClusterIPEndpointsMore as TestClusterIPGeneral Create some ClusterIP services and use runPacketFlowTests to test general functionality: - normal connection - hairpin connection - multiple endpoints - port != targetPort - multiple protocols on same port Remove the assertIPTablesRulesEqual test because the packet flow tests cover all of the details we care about here. --- pkg/proxy/iptables/proxier_test.go | 187 ++++++++++++++++++++--------- 1 file changed, 131 insertions(+), 56 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index f13dd511edd..34a08b941f8 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -2081,84 +2081,111 @@ func TestClusterIPReject(t *testing.T) { }) } -func TestClusterIPEndpointsMore(t *testing.T) { +// TestClusterIPGeneral tests various basic features of a ClusterIP service +func TestClusterIPGeneral(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) - svcIP := "172.30.0.41" - svcPort := 80 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolSCTP, - } makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.ClusterIP = svcIP + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.ClusterIP = "172.30.0.41" svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolSCTP, + Name: "http", + Port: 80, + Protocol: v1.ProtocolTCP, }} }), + makeTestService("ns2", "svc2", func(svc *v1.Service) { + svc.Spec.ClusterIP = "172.30.0.42" + svc.Spec.Ports = []v1.ServicePort{ + { + Name: "http", + Port: 80, + Protocol: v1.ProtocolTCP, + }, + { + Name: "https", + Port: 443, + Protocol: v1.ProtocolTCP, + TargetPort: intstr.FromInt32(8443), + }, + { + // Of course this should really be UDP, but if we + // create a service with UDP ports, the Proxier will + // try to do conntrack cleanup and we'd have to set + // the FakeExec up to be able to deal with that... + Name: "dns-sctp", + Port: 53, + Protocol: v1.ProtocolSCTP, + }, + { + Name: "dns-tcp", + Port: 53, + Protocol: v1.ProtocolTCP, + // We use TargetPort on TCP but not SCTP to help + // disambiguate the output. + TargetPort: intstr.FromInt32(5353), + }, + } + }), ) - epIP := "10.180.0.1" populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { + makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP}, + Addresses: []string{"10.180.0.1"}, + NodeName: pointer.String(testHostname), }} eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &sctpProtocol, + Name: pointer.String("http"), + Port: pointer.Int32(80), + Protocol: &tcpProtocol, }} }), + makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) { + eps.AddressType = discovery.AddressTypeIPv4 + eps.Endpoints = []discovery.Endpoint{ + { + Addresses: []string{"10.180.0.1"}, + NodeName: pointer.String(testHostname), + }, + { + Addresses: []string{"10.180.2.1"}, + NodeName: pointer.String("host2"), + }, + } + eps.Ports = []discovery.EndpointPort{ + { + Name: pointer.String("http"), + Port: pointer.Int32(80), + Protocol: &tcpProtocol, + }, + { + Name: pointer.String("https"), + Port: pointer.Int32(8443), + Protocol: &tcpProtocol, + }, + { + Name: pointer.String("dns-sctp"), + Port: pointer.Int32(53), + Protocol: &sctpProtocol, + }, + { + Name: pointer.String("dns-tcp"), + Port: pointer.Int32(5353), + Protocol: &tcpProtocol, + }, + } + }), ) fp.syncProxyRules() - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-RFW33Y6OHVBQ4W3M - [0:0] - :KUBE-SVC-GFCIFIA5VTFSTMSM - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m sctp -p sctp -d 172.30.0.41 --dport 80 -j KUBE-SVC-GFCIFIA5VTFSTMSM - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-RFW33Y6OHVBQ4W3M -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-RFW33Y6OHVBQ4W3M -m comment --comment ns1/svc1:p80 -m sctp -p sctp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SVC-GFCIFIA5VTFSTMSM -m comment --comment "ns1/svc1:p80 cluster IP" -m sctp -p sctp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-GFCIFIA5VTFSTMSM -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-RFW33Y6OHVBQ4W3M - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { - name: "cluster IP accepted", + name: "simple clusterIP", sourceIP: "10.180.0.2", - protocol: v1.ProtocolSCTP, destIP: "172.30.0.41", destPort: 80, output: "10.180.0.1:80", @@ -2167,12 +2194,60 @@ func TestClusterIPEndpointsMore(t *testing.T) { { name: "hairpin to cluster IP", sourceIP: "10.180.0.1", - protocol: v1.ProtocolSCTP, destIP: "172.30.0.41", destPort: 80, output: "10.180.0.1:80", masq: true, }, + { + name: "clusterIP with multiple endpoints", + sourceIP: "10.180.0.2", + destIP: "172.30.0.42", + destPort: 80, + output: "10.180.0.1:80, 10.180.2.1:80", + masq: false, + }, + { + name: "clusterIP with TargetPort", + sourceIP: "10.180.0.2", + destIP: "172.30.0.42", + destPort: 443, + output: "10.180.0.1:8443, 10.180.2.1:8443", + masq: false, + }, + { + name: "clusterIP with TCP and SCTP on same port (TCP)", + sourceIP: "10.180.0.2", + protocol: v1.ProtocolTCP, + destIP: "172.30.0.42", + destPort: 53, + output: "10.180.0.1:5353, 10.180.2.1:5353", + masq: false, + }, + { + name: "clusterIP with TCP and SCTP on same port (SCTP)", + sourceIP: "10.180.0.2", + protocol: v1.ProtocolSCTP, + destIP: "172.30.0.42", + destPort: 53, + output: "10.180.0.1:53, 10.180.2.1:53", + masq: false, + }, + { + name: "TCP-only port does not match UDP traffic", + sourceIP: "10.180.0.2", + protocol: v1.ProtocolUDP, + destIP: "172.30.0.42", + destPort: 80, + output: "", + }, + { + name: "svc1 does not accept svc2's ports", + sourceIP: "10.180.0.2", + destIP: "172.30.0.41", + destPort: 443, + output: "", + }, }) } From f38231d5687239ca2a27f5a7ad4c04aaca558772 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Mon, 26 Jun 2023 06:48:56 -0400 Subject: [PATCH 09/13] Merge all the "reject when no endpoints" tests together Merge TestClusterIPReject, TestExternalIPsReject, TestNodePortReject, and TestLoadBalancerReject into a single test. Also remove the assertIPTablesRulesEqual tests because the packet flow tests cover all of the details we care about here. --- pkg/proxy/iptables/proxier_test.go | 333 +++++------------------------ 1 file changed, 51 insertions(+), 282 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 34a08b941f8..97089dc16d3 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -2019,11 +2019,16 @@ func TestOverallIPTablesRules(t *testing.T) { } } -func TestClusterIPReject(t *testing.T) { +// TestNoEndpointsReject tests that a service with no endpoints rejects connections to +// its ClusterIP, ExternalIPs, NodePort, and LoadBalancer IP. +func TestNoEndpointsReject(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) svcIP := "172.30.0.41" svcPort := 80 + svcNodePort := 3001 + svcExternalIPs := "192.168.99.11" + svcLBIP := "1.2.3.4" svcPortName := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc1"), Port: "p80", @@ -2031,51 +2036,63 @@ func TestClusterIPReject(t *testing.T) { makeServiceMap(fp, makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ClusterIP = svcIP + svc.Spec.ExternalIPs = []string{svcExternalIPs} svc.Spec.Ports = []v1.ServicePort{{ Name: svcPortName.Port, - Port: int32(svcPort), Protocol: v1.ProtocolTCP, + Port: int32(svcPort), + NodePort: int32(svcNodePort), + }} + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: svcLBIP, }} }), ) fp.syncProxyRules() - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `) - - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { - name: "cluster IP rejected", + name: "pod to cluster IP with no endpoints", sourceIP: "10.0.0.2", - destIP: "172.30.0.41", - destPort: 80, + destIP: svcIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "external to external IP with no endpoints", + sourceIP: testExternalClient, + destIP: svcExternalIPs, + destPort: svcPort, + output: "REJECT", + }, + { + name: "pod to NodePort with no endpoints", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: "REJECT", + }, + { + name: "external to NodePort with no endpoints", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: "REJECT", + }, + { + name: "pod to LoadBalancer IP with no endpoints", + sourceIP: "10.0.0.2", + destIP: svcLBIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "external to LoadBalancer IP with no endpoints", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, output: "REJECT", }, }) @@ -2718,80 +2735,6 @@ func TestMasqueradeRule(t *testing.T) { } } -func TestExternalIPsReject(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "172.30.0.41" - svcPort := 80 - svcExternalIPs := "192.168.99.11" - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "ClusterIP" - svc.Spec.ClusterIP = svcIP - svc.Spec.ExternalIPs = []string{svcExternalIPs} - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - TargetPort: intstr.FromInt32(int32(svcPort)), - }} - }), - ) - - fp.syncProxyRules() - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j REJECT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ - { - name: "cluster IP with no endpoints", - sourceIP: "10.0.0.2", - destIP: svcIP, - destPort: svcPort, - output: "REJECT", - }, - { - name: "external IP with no endpoints", - sourceIP: testExternalClient, - destIP: svcExternalIPs, - destPort: svcPort, - output: "REJECT", - }, - }) -} - func TestOnlyLocalExternalIPs(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) @@ -3013,180 +2956,6 @@ func TestNonLocalExternalIPs(t *testing.T) { }) } -func TestNodePortReject(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "172.30.0.41" - svcPort := 80 - svcNodePort := 3001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) - - fp.syncProxyRules() - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to cluster IP", - sourceIP: "10.0.0.2", - destIP: svcIP, - destPort: svcPort, - output: "REJECT", - }, - { - name: "pod to NodePort", - sourceIP: "10.0.0.2", - destIP: testNodeIP, - destPort: svcNodePort, - output: "REJECT", - }, - { - name: "external to NodePort", - sourceIP: testExternalClient, - destIP: testNodeIP, - destPort: svcNodePort, - output: "REJECT", - }, - }) -} - -func TestLoadBalancerReject(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "172.30.0.41" - svcPort := 80 - svcNodePort := 3001 - svcHealthCheckNodePort := 30000 - svcLBIP := "1.2.3.4" - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - svcSessionAffinityTimeout := int32(10800) - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "LoadBalancer" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.HealthCheckNodePort = int32(svcHealthCheckNodePort) - svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ - IP: svcLBIP, - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP - svc.Spec.SessionAffinityConfig = &v1.SessionAffinityConfig{ - ClientIP: &v1.ClientIPConfig{TimeoutSeconds: &svcSessionAffinityTimeout}, - } - }), - ) - - fp.syncProxyRules() - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j REJECT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j REJECT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to cluster IP", - sourceIP: "10.0.0.2", - destIP: svcIP, - destPort: svcPort, - output: "REJECT", - }, - { - name: "pod to LoadBalancer IP", - sourceIP: "10.0.0.2", - destIP: svcLBIP, - destPort: svcPort, - output: "REJECT", - }, - { - name: "external to LoadBalancer IP", - sourceIP: testExternalClient, - destIP: svcLBIP, - destPort: svcPort, - output: "REJECT", - }, - }) -} - func TestOnlyLocalLoadBalancing(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) From ff5f5bc1613bb7c915ce821e3fd07032c7396fb0 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 7 Jul 2023 12:17:56 -0400 Subject: [PATCH 10/13] Merge several NodePort tests into TestNodePorts Previously we had TestNodePort, which tested basic NodePort behavior, plus Test{Enable,Disable}LocalhostNodePorts{IPv4,IPv6} to test the behavior of --localhost-nodeports under IPv4 and IPv6, plus TestDisableLocalhostNodePortsIPv4WithNodeAddress to test --nodeport-addresses. Merge all of these together into TestNodePorts, and use runPacketFlowTests to check the results rather than assertIPTablesRulesEqual. The packet tracer is not full-featured enough to be able to check the "anti martian packet spoofing" rule, so we check the iptables dump for that manually. (This also fixes the --localhost-nodeport tests to use the same IP ranges as most of the other tests now.) --- pkg/proxy/iptables/proxier_test.go | 831 +++++++++-------------------- 1 file changed, 246 insertions(+), 585 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 97089dc16d3..12fa88d2324 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -304,7 +304,7 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { podCIDR := "10.0.0.0/8" if ipt.IsIPv6() { ipfamily = v1.IPv6Protocol - podCIDR = "fd00::/64" + podCIDR = "fd00:10::/64" } detectLocal, _ := proxyutiliptables.NewDetectLocalByCIDR(podCIDR) @@ -2480,128 +2480,257 @@ func TestLoadBalancer(t *testing.T) { }) } -func TestNodePort(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "172.30.0.41" - svcPort := 80 - svcNodePort := 3001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, +// TestNodePorts tests NodePort services under various combinations of the +// --nodeport-addresses and --localhost-nodeports flags. +func TestNodePorts(t *testing.T) { + testCases := []struct { + name string + + family v1.IPFamily + localhostNodePorts bool + nodePortAddresses []string + + // allowAltNodeIP is true if we expect NodePort traffic on the alternate + // node IP to be accepted + allowAltNodeIP bool + + // expectFirewall is true if we expect KUBE-FIREWALL to be filled in with + // an anti-martian-packet rule + expectFirewall bool + }{ + { + name: "ipv4, localhost-nodeports enabled", + + family: v1.IPv4Protocol, + localhostNodePorts: true, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: true, + }, + { + name: "ipv4, localhost-nodeports disabled", + + family: v1.IPv4Protocol, + localhostNodePorts: false, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv4, localhost-nodeports disabled, localhost in nodeport-addresses", + + family: v1.IPv4Protocol, + localhostNodePorts: false, + nodePortAddresses: []string{"192.168.0.0/24", "127.0.0.1/32"}, + + allowAltNodeIP: false, + expectFirewall: false, + }, + { + name: "ipv4, localhost-nodeports enabled, multiple nodeport-addresses", + + family: v1.IPv4Protocol, + localhostNodePorts: false, + nodePortAddresses: []string{"192.168.0.0/24", "192.168.1.0/24", "2001:db8::/64"}, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv6, localhost-nodeports enabled", + + family: v1.IPv6Protocol, + localhostNodePorts: true, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv6, localhost-nodeports disabled", + + family: v1.IPv6Protocol, + localhostNodePorts: false, + nodePortAddresses: nil, + + allowAltNodeIP: true, + expectFirewall: false, + }, + { + name: "ipv6, localhost-nodeports disabled, multiple nodeport-addresses", + + family: v1.IPv6Protocol, + localhostNodePorts: false, + nodePortAddresses: []string{"192.168.0.0/24", "192.168.1.0/24", "2001:db8::/64"}, + + allowAltNodeIP: false, + expectFirewall: false, + }, } - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ipt *iptablestest.FakeIPTables + var svcIP, epIP1, epIP2 string + if tc.family == v1.IPv4Protocol { + ipt = iptablestest.NewFake() + svcIP = "172.30.0.41" + epIP1 = "10.180.0.1" + epIP2 = "10.180.2.1" + } else { + ipt = iptablestest.NewIPv6Fake() + svcIP = "fd00:172:30::41" + epIP1 = "fd00:10:180::1" + epIP2 = "fd00:10:180::2:1" + } + fp := NewFakeProxier(ipt) + fp.localhostNodePorts = tc.localhostNodePorts + if tc.nodePortAddresses != nil { + fp.nodePortAddresses = proxyutil.NewNodePortAddresses(tc.family, tc.nodePortAddresses) + } - epIP := "10.180.0.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP}, - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) + makeServiceMap(fp, + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeNodePort + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: 3001, + }} + }), + ) - fp.syncProxyRules() + populateEndpointSlices(fp, + makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { + if tc.family == v1.IPv4Protocol { + eps.AddressType = discovery.AddressTypeIPv4 + } else { + eps.AddressType = discovery.AddressTypeIPv6 + } + eps.Endpoints = []discovery.Endpoint{{ + Addresses: []string{epIP1}, + NodeName: nil, + }, { + Addresses: []string{epIP2}, + NodeName: pointer.String(testHostname), + }} + eps.Ports = []discovery.EndpointPort{{ + Name: pointer.String("p80"), + Port: pointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), + ) - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) + fp.syncProxyRules() - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to cluster IP", - sourceIP: "10.0.0.2", - destIP: svcIP, - destPort: svcPort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: false, - }, - { - name: "external to nodePort", - sourceIP: testExternalClient, - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - { - name: "external to nodePort on secondary IP", - sourceIP: testExternalClient, - destIP: testNodeIPAlt, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - { - name: "node to nodePort", - sourceIP: testNodeIP, - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - { - name: "localhost to nodePort gets masqueraded", - sourceIP: "127.0.0.1", - destIP: "127.0.0.1", - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP, svcPort), - masq: true, - }, - }) + var podIP, externalClientIP, nodeIP, altNodeIP, localhostIP string + if tc.family == v1.IPv4Protocol { + podIP = "10.0.0.2" + externalClientIP = testExternalClient + nodeIP = testNodeIP + altNodeIP = testNodeIPAlt + localhostIP = "127.0.0.1" + } else { + podIP = "fd00:10::2" + externalClientIP = "2600:5200::1" + nodeIP = testNodeIPv6 + altNodeIP = testNodeIPv6Alt + localhostIP = "::1" + } + output := net.JoinHostPort(epIP1, "80") + ", " + net.JoinHostPort(epIP2, "80") + + // Basic tests are the same for all cases + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: podIP, + destIP: svcIP, + destPort: 80, + output: output, + masq: false, + }, + { + name: "external to nodePort", + sourceIP: externalClientIP, + destIP: nodeIP, + destPort: 3001, + output: output, + masq: true, + }, + { + name: "node to nodePort", + sourceIP: nodeIP, + destIP: nodeIP, + destPort: 3001, + output: output, + masq: true, + }, + }) + + // localhost to NodePort is only allowed in IPv4, and only if not disabled + if tc.family == v1.IPv4Protocol && tc.localhostNodePorts { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "localhost to nodePort gets masqueraded", + sourceIP: localhostIP, + destIP: localhostIP, + destPort: 3001, + output: output, + masq: true, + }, + }) + } else { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "localhost to nodePort is ignored", + sourceIP: localhostIP, + destIP: localhostIP, + destPort: 3001, + output: "", + }, + }) + } + + // NodePort on altNodeIP should be allowed, unless + // nodePortAddressess excludes altNodeIP + if tc.allowAltNodeIP { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "external to nodePort on secondary IP", + sourceIP: externalClientIP, + destIP: altNodeIP, + destPort: 3001, + output: output, + masq: true, + }, + }) + } else { + runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ + { + name: "secondary nodeIP ignores NodePorts", + sourceIP: externalClientIP, + destIP: altNodeIP, + destPort: 3001, + output: "", + }, + }) + } + + // We have to check the firewall rule manually rather than via + // runPacketFlowTests(), because the packet tracer doesn't + // implement conntrack states. + var expected string + if tc.expectFirewall { + expected = "-A KUBE-FIREWALL -m comment --comment \"block incoming localnet connections\" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP\n" + } + assertIPTablesChainEqual(t, getLine(), utiliptables.TableFilter, kubeletFirewallChain, expected, fp.iptablesData.String()) + }) + } } func TestHealthCheckNodePort(t *testing.T) { @@ -3093,474 +3222,6 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { }) } -func TestEnableLocalhostNodePortsIPv4(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = true - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-6KG6DFHVBKBK53RU - [0:0] - :KUBE-SEP-KDGX2M2ONE25PSWH - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.69.0.10 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -s 10.244.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.0.1:80 - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -s 10.244.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-6KG6DFHVBKBK53RU - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - COMMIT - `) - svcIP := "10.69.0.10" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "10.244.0.1" - epIP2 := "10.244.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestDisableLocalhostNodePortsIPv4(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = false - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-6KG6DFHVBKBK53RU - [0:0] - :KUBE-SEP-KDGX2M2ONE25PSWH - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.69.0.10 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL ! -d 127.0.0.0/8 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -s 10.244.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.0.1:80 - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -s 10.244.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-6KG6DFHVBKBK53RU - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - COMMIT - `) - svcIP := "10.69.0.10" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "10.244.0.1" - epIP2 := "10.244.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestDisableLocalhostNodePortsIPv4WithNodeAddress(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = false - fp.networkInterfacer.InterfaceAddrs() - fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"127.0.0.0/8"}) - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-6KG6DFHVBKBK53RU - [0:0] - :KUBE-SEP-KDGX2M2ONE25PSWH - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.69.0.10 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -s 10.244.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-6KG6DFHVBKBK53RU -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.0.1:80 - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -s 10.244.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-KDGX2M2ONE25PSWH -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.244.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-6KG6DFHVBKBK53RU - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.244.2.1:80" -j KUBE-SEP-KDGX2M2ONE25PSWH - COMMIT - `) - svcIP := "10.69.0.10" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "10.244.0.1" - epIP2 := "10.244.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestEnableLocalhostNodePortsIPv6(t *testing.T) { - ipt := iptablestest.NewIPv6Fake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = true - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-LIGRYQQLSZN4UWQ5 - [0:0] - :KUBE-SEP-XJJ5QXWGJG344QDZ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d fd00:ab34::20 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL ! -d ::1/128 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -s ff06::c1 -j KUBE-MARK-MASQ - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c1]:80 - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -s ff06::c2 -j KUBE-MARK-MASQ - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c2]:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c1]:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-LIGRYQQLSZN4UWQ5 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - COMMIT - `) - svcIP := "fd00:ab34::20" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "ff06::c1" - epIP2 := "ff06::c2" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv6 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - -func TestDisableLocalhostNodePortsIPv6(t *testing.T) { - ipt := iptablestest.NewIPv6Fake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.localhostNodePorts = false - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-LIGRYQQLSZN4UWQ5 - [0:0] - :KUBE-SEP-XJJ5QXWGJG344QDZ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 30001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d fd00:ab34::20 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL ! -d ::1/128 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -s ff06::c1 -j KUBE-MARK-MASQ - -A KUBE-SEP-LIGRYQQLSZN4UWQ5 -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c1]:80 - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -s ff06::c2 -j KUBE-MARK-MASQ - -A KUBE-SEP-XJJ5QXWGJG344QDZ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination [ff06::c2]:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c1]:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-LIGRYQQLSZN4UWQ5 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> [ff06::c2]:80" -j KUBE-SEP-XJJ5QXWGJG344QDZ - COMMIT - `) - svcIP := "fd00:ab34::20" - svcPort := 80 - svcNodePort := 30001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "ff06::c1" - epIP2 := "ff06::c2" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv6 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) -} - func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) From 19f19e2f4f904233a71e109a5fbfaa112600b8bc Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Mon, 26 Jun 2023 07:21:19 -0400 Subject: [PATCH 11/13] Merge the `ExternalTrafficPolicy: Local` tests together Merge TestOnlyLocalExternalIPs, TestOnlyLocalLoadBalancing, and TestOnlyLocalNodePorts together into TestExternalTrafficPolicyLocal. Drop the assertIPTablesRulesEqual tests in favor of runPacketFlowTests. Remove TestOnlyLocalNodePortsNoClusterCIDR; the relevant bits of the "no local detector" case are already fully covered by TestInternalExternalMasquerade. --- pkg/proxy/iptables/proxier_test.go | 448 ++++------------------------- 1 file changed, 58 insertions(+), 390 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 12fa88d2324..48ababb2ba9 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -2864,12 +2864,20 @@ func TestMasqueradeRule(t *testing.T) { } } -func TestOnlyLocalExternalIPs(t *testing.T) { +// TestExternalTrafficPolicyLocal tests that traffic to externally-facing IPs does not get +// masqueraded when using Local traffic policy. For traffic from external sources, that +// means it can also only be routed to local endpoints, but for traffic from internal +// sources, it gets routed to all endpoints. +func TestExternalTrafficPolicyLocal(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) + svcIP := "172.30.0.41" svcPort := 80 + svcNodePort := 3001 + svcHealthCheckNodePort := 30000 svcExternalIPs := "192.168.99.11" + svcLBIP := "1.2.3.4" svcPortName := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc1"), Port: "p80", @@ -2877,7 +2885,7 @@ func TestOnlyLocalExternalIPs(t *testing.T) { makeServiceMap(fp, makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" + svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal svc.Spec.ClusterIP = svcIP svc.Spec.ExternalIPs = []string{svcExternalIPs} @@ -2885,10 +2893,16 @@ func TestOnlyLocalExternalIPs(t *testing.T) { Name: svcPortName.Port, Port: int32(svcPort), Protocol: v1.ProtocolTCP, + NodePort: int32(svcNodePort), TargetPort: intstr.FromInt32(int32(svcPort)), }} + svc.Spec.HealthCheckNodePort = int32(svcHealthCheckNodePort) + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: svcLBIP, + }} }), ) + epIP1 := "10.180.0.1" epIP2 := "10.180.2.1" populateEndpointSlices(fp, @@ -2910,55 +2924,9 @@ func TestOnlyLocalExternalIPs(t *testing.T) { fp.syncProxyRules() - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { - name: "cluster IP hits both endpoints", + name: "pod to cluster IP hits both endpoints, unmasqueraded", sourceIP: "10.0.0.2", destIP: svcIP, destPort: svcPort, @@ -2966,13 +2934,53 @@ func TestOnlyLocalExternalIPs(t *testing.T) { masq: false, }, { - name: "external IP hits only local endpoint, unmasqueraded", + name: "pod to external IP hits both endpoints, unmasqueraded", + sourceIP: "10.0.0.2", + destIP: svcExternalIPs, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to external IP hits only local endpoint, unmasqueraded", sourceIP: testExternalClient, destIP: svcExternalIPs, destPort: svcPort, output: fmt.Sprintf("%s:%d", epIP2, svcPort), masq: false, }, + { + name: "pod to LB IP hits only both endpoints, unmasqueraded", + sourceIP: "10.0.0.2", + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to LB IP hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + { + name: "pod to NodePort hits both endpoints, unmasqueraded", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to NodePort hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, }) } @@ -3085,346 +3093,6 @@ func TestNonLocalExternalIPs(t *testing.T) { }) } -func TestOnlyLocalLoadBalancing(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "172.30.0.41" - svcPort := 80 - svcNodePort := 3001 - svcHealthCheckNodePort := 30000 - svcLBIP := "1.2.3.4" - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - svcSessionAffinityTimeout := int32(10800) - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "LoadBalancer" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.HealthCheckNodePort = int32(svcHealthCheckNodePort) - svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ - IP: svcLBIP, - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP - svc.Spec.SessionAffinityConfig = &v1.SessionAffinityConfig{ - ClientIP: &v1.ClientIPConfig{TimeoutSeconds: &svcSessionAffinityTimeout}, - } - }), - ) - - epIP1 := "10.180.0.1" - epIP2 := "10.180.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to cluster IP hits both endpoints", - sourceIP: "10.0.0.2", - destIP: svcIP, - destPort: svcPort, - output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), - masq: false, - }, - { - name: "external to LB IP hits only local endpoint, unmasqueraded", - sourceIP: testExternalClient, - destIP: svcLBIP, - destPort: svcPort, - output: fmt.Sprintf("%s:%d", epIP2, svcPort), - masq: false, - }, - { - name: "external to NodePort hits only local endpoint, unmasqueraded", - sourceIP: testExternalClient, - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP2, svcPort), - masq: false, - }, - }) -} - -func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() - fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"192.168.0.0/24", "2001:db8::/64"}) - fp.localhostNodePorts = false - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - COMMIT - `) - onlyLocalNodePorts(t, fp, ipt, expected, getLine()) -} - -func TestOnlyLocalNodePorts(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"192.168.0.0/24", "2001:db8::/64"}) - fp.localhostNodePorts = false - - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - -A KUBE-SVL-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - COMMIT - `) - onlyLocalNodePorts(t, fp, ipt, expected, getLine()) -} - -func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTables, expected string, line int) { - svcIP := "172.30.0.41" - svcPort := 80 - svcNodePort := 3001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal - }), - ) - - epIP1 := "10.180.0.1" - epIP2 := "10.180.2.1" - populateEndpointSlices(fp, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIP1}, - NodeName: nil, - }, { - Addresses: []string{epIP2}, - NodeName: pointer.String(testHostname), - }} - eps.Ports = []discovery.EndpointPort{{ - Name: pointer.String(svcPortName.Port), - Port: pointer.Int32(int32(svcPort)), - Protocol: &tcpProtocol, - }} - }), - ) - - fp.syncProxyRules() - - assertIPTablesRulesEqual(t, line, true, expected, fp.iptablesData.String()) - - runPacketFlowTests(t, line, ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to cluster IP hit both endpoints", - sourceIP: "10.0.0.2", - destIP: svcIP, - destPort: svcPort, - output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), - masq: false, - }, - { - name: "external to NodePort hits only local endpoint", - sourceIP: testExternalClient, - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP2, svcPort), - masq: false, - }, - { - name: "pod to localhost doesn't work because localhost is not in nodePortAddresses", - sourceIP: "10.0.0.2", - destIP: "127.0.0.1", - destPort: svcNodePort, - output: "", - }, - }) - - if fp.localDetector.IsImplemented() { - // pod-to-NodePort is treated as internal traffic, so we see both endpoints - runPacketFlowTests(t, line, ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to NodePort hits both endpoints", - sourceIP: "10.0.0.2", - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), - masq: false, - }, - }) - } else { - // pod-to-NodePort is (incorrectly) treated as external traffic - // when there is no LocalTrafficDetector. - runPacketFlowTests(t, line, ipt, testNodeIPs, []packetFlowTest{ - { - name: "pod to NodePort hits only local endpoint", - sourceIP: "10.0.0.2", - destIP: testNodeIP, - destPort: svcNodePort, - output: fmt.Sprintf("%s:%d", epIP2, svcPort), - masq: false, - }, - }) - } -} - func TestComputeProbability(t *testing.T) { expectedProbabilities := map[int]string{ 1: "1.0000000000", From de077f448e923db017dc993d28e08df0ec996fa4 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 22 Sep 2023 10:54:40 -0400 Subject: [PATCH 12/13] Rename TestNonLocalExternalIPs to TestExternalTrafficPolicyCluster For consistency with TestExternalTrafficPolicyLocal, test all of the Cluster external traffic policy cases together here (ensuring that masquerading happens where needed). Drop the assertIPTablesRulesEqual test in favor of runPacketFlowTests. --- pkg/proxy/iptables/proxier_test.go | 102 ++++++++++++++++------------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 48ababb2ba9..1d3e23d1903 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -2984,14 +2984,17 @@ func TestExternalTrafficPolicyLocal(t *testing.T) { }) } -// TestNonLocalExternalIPs tests if we add the masquerade rule into svcChain in order to -// SNAT packets to external IPs if externalTrafficPolicy is cluster and the traffic is NOT Local. -func TestNonLocalExternalIPs(t *testing.T) { +// TestExternalTrafficPolicyCluster tests that traffic to an externally-facing IP gets +// masqueraded when using Cluster traffic policy. +func TestExternalTrafficPolicyCluster(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) + svcIP := "172.30.0.41" svcPort := 80 + svcNodePort := 3001 svcExternalIPs := "192.168.99.11" + svcLBIP := "1.2.3.4" svcPortName := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc1"), Port: "p80", @@ -2999,16 +3002,23 @@ func TestNonLocalExternalIPs(t *testing.T) { makeServiceMap(fp, makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeLoadBalancer svc.Spec.ClusterIP = svcIP svc.Spec.ExternalIPs = []string{svcExternalIPs} svc.Spec.Ports = []v1.ServicePort{{ Name: svcPortName.Port, Port: int32(svcPort), Protocol: v1.ProtocolTCP, + NodePort: int32(svcNodePort), TargetPort: intstr.FromInt32(int32(svcPort)), }} + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: svcLBIP, + }} + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyCluster }), ) + epIP1 := "10.180.0.1" epIP2 := "10.180.2.1" populateEndpointSlices(fp, @@ -3031,51 +3041,9 @@ func TestNonLocalExternalIPs(t *testing.T) { fp.syncProxyRules() - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.2.1:80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { - name: "pod to cluster IP", + name: "pod to cluster IP hits both endpoints, unmasqueraded", sourceIP: "10.0.0.2", destIP: svcIP, destPort: svcPort, @@ -3083,13 +3051,53 @@ func TestNonLocalExternalIPs(t *testing.T) { masq: false, }, { - name: "external to external IP", + name: "pod to external IP hits both endpoints, masqueraded", + sourceIP: "10.0.0.2", + destIP: svcExternalIPs, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: true, + }, + { + name: "external to external IP hits both endpoints, masqueraded", sourceIP: testExternalClient, destIP: svcExternalIPs, destPort: svcPort, output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), masq: true, }, + { + name: "pod to LB IP hits both endpoints, masqueraded", + sourceIP: "10.0.0.2", + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: true, + }, + { + name: "external to LB IP hits both endpoints, masqueraded", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: true, + }, + { + name: "pod to NodePort hits both endpoints, masqueraded", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: true, + }, + { + name: "external to NodePort hits both endpoints, masqueraded", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: true, + }, }) } From 2b973806bc786a44cb9f5aa1aad4796e9d33f918 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 22 Sep 2023 11:45:32 -0400 Subject: [PATCH 13/13] Remove remaining unnecessary assertIPTablesRulesEqual checks TestLoadBalancer and TestHealthCheckNodePort still had iptables rules checks, but they also have sufficient runPacketFlowTests checks to cover everything we care about. (This leaves only TestOverallIPTablesRules and TestSyncProxyRulesRepeated using assertIPTablesRulesEqual.) --- pkg/proxy/iptables/proxier_test.go | 80 ------------------------------ 1 file changed, 80 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 1d3e23d1903..6dc6a2c8335 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -2322,55 +2322,6 @@ func TestLoadBalancer(t *testing.T) { fp.syncProxyRules() - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - -A KUBE-PROXY-FIREWALL -m comment --comment "ns1/svc1:p80 traffic not accepted by KUBE-FW-XPGD46QRK7WJZT7O" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP - -A KUBE-PROXY-FIREWALL -m comment --comment "ns1/svc1:p80 traffic not accepted by KUBE-FW-XPGD46QRK7WJZT7O" -m tcp -p tcp -d 5.6.7.8 --dport 80 -j DROP - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-FW-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 5.6.7.8 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 192.168.0.0/24 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 1.2.3.4 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 5.6.7.8 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "other traffic to ns1/svc1:p80 will be dropped by KUBE-PROXY-FIREWALL" - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ - COMMIT - `) - - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "pod to cluster IP", @@ -2763,37 +2714,6 @@ func TestHealthCheckNodePort(t *testing.T) { makeServiceMap(fp, svc) fp.syncProxyRules() - expected := dedent.Dedent(` - *filter - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FIREWALL - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-PROXY-FIREWALL - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j REJECT - -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT - -A KUBE-FIREWALL -m comment --comment "block incoming localnet connections" -d 127.0.0.0/8 ! -s 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-POSTROUTING - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 127.0.0.1 -j KUBE-NODEPORTS - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - COMMIT - `) - - assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String()) - runPacketFlowTests(t, getLine(), ipt, testNodeIPs, []packetFlowTest{ { name: "firewall accepts HealthCheckNodePort",