From 261f4f9e2d7f09230c68c43db00b61540067924a Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sat, 19 Mar 2022 13:47:14 -0400 Subject: [PATCH 1/5] proxy/iptables: remove redundant test We originally had one HealthCheckNodePort test that used assertIPTablesRulesEqual() and one that didn't, but later I went through and made all the tests use assertIPTablesRulesEqual() and didn't notice that this resulted in there now being two nearly-identical HealthCheckNodePort tests. --- pkg/proxy/iptables/proxier_test.go | 109 ----------------------------- 1 file changed, 109 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index c2c94981d08..faa3b7d23df 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -3900,115 +3900,6 @@ func TestEndpointSliceE2E(t *testing.T) { assertIPTablesRulesNotEqual(t, getLine(), expectedIPTablesWithSlice, fp.iptablesData.String()) } -func TestHealthCheckNodePortE2E(t *testing.T) { - expectedIPTables := dedent.Dedent(` - *filter - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-EXT-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-NODEPORTS - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] - :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] - :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1 -m tcp -p tcp --dport 30010 -j KUBE-EXT-AQI2S6QIMU7PVVRP - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ - -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2 -j KUBE-MARK-MASQ - -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3 -j KUBE-MARK-MASQ - -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ - -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -j KUBE-SEP-3JOIVZTXZZRGORX4 - COMMIT - `) - - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.OnServiceSynced() - fp.OnEndpointSlicesSynced() - - serviceName := "svc1" - namespaceName := "ns1" - - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, - Spec: v1.ServiceSpec{ - ClusterIP: "172.30.1.1", - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt(80), NodePort: 30010, Protocol: v1.ProtocolTCP}}, - Type: "LoadBalancer", - HealthCheckNodePort: 30000, - ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, - }, - } - fp.OnServiceAdd(svc) - - tcpProtocol := v1.ProtocolTCP - endpointSlice := &discovery.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-1", serviceName), - Namespace: namespaceName, - Labels: map[string]string{discovery.LabelServiceName: serviceName}, - }, - Ports: []discovery.EndpointPort{{ - Name: utilpointer.StringPtr(""), - Port: utilpointer.Int32Ptr(80), - Protocol: &tcpProtocol, - }}, - AddressType: discovery.AddressTypeIPv4, - Endpoints: []discovery.Endpoint{{ - Addresses: []string{"10.0.1.1"}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, - NodeName: utilpointer.StringPtr(testHostname), - }, { - Addresses: []string{"10.0.1.2"}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, - NodeName: utilpointer.StringPtr("node2"), - }, { - Addresses: []string{"10.0.1.3"}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, - NodeName: utilpointer.StringPtr("node3"), - }, { - Addresses: []string{"10.0.1.4"}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(false)}, - NodeName: utilpointer.StringPtr("node4"), - }}, - } - fp.OnEndpointSliceAdd(endpointSlice) - fp.syncProxyRules() - assertIPTablesRulesEqual(t, getLine(), expectedIPTables, fp.iptablesData.String()) - - fp.OnServiceDelete(svc) - fp.syncProxyRules() - assertIPTablesRulesNotEqual(t, getLine(), expectedIPTables, fp.iptablesData.String()) -} - // TestHealthCheckNodePortWhenTerminating tests that health check node ports are not enabled when all local endpoints are terminating func TestHealthCheckNodePortWhenTerminating(t *testing.T) { ipt := iptablestest.NewFake() From d9ca6655512f9a9d58f3d5683bd137771bd8dbe4 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sat, 19 Mar 2022 13:53:45 -0400 Subject: [PATCH 2/5] proxy/iptables: fix up some test case comments --- pkg/proxy/iptables/proxier_test.go | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index faa3b7d23df..0e6aeab1b82 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -4314,7 +4314,7 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { expectedIPTablesWithSlice: clusterExpectedIPTables, }, { - name: "internalTrafficPolicy is local and there is non-zero local endpoints", + name: "internalTrafficPolicy is local and there are local endpoints", line: getLine(), internalTrafficPolicy: &local, featureGateOn: true, @@ -4355,7 +4355,7 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { `), }, { - name: "internalTrafficPolicy is local and there is zero local endpoint", + name: "internalTrafficPolicy is local and there are no local endpoints", line: getLine(), internalTrafficPolicy: &local, featureGateOn: true, @@ -4393,7 +4393,7 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { `), }, { - name: "internalTrafficPolicy is local and there is non-zero local endpoint with feature gate off", + name: "Local internalTrafficPolicy is ignored when feature gate is off", line: getLine(), internalTrafficPolicy: &local, featureGateOn: false, @@ -4548,7 +4548,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { NodeName: utilpointer.StringPtr(testHostname), }, { - // this endpoint should be ignored for node ports since there are ready non-terminating endpoints + // this endpoint should be ignored for external since there are ready non-terminating endpoints Addresses: []string{"10.0.1.3"}, Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(false), @@ -4558,7 +4558,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { NodeName: utilpointer.StringPtr(testHostname), }, { - // this endpoint should be ignored for node ports since there are ready non-terminating endpoints + // this endpoint should be ignored for external since there are ready non-terminating endpoints Addresses: []string{"10.0.1.4"}, Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(false), @@ -4568,7 +4568,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { NodeName: utilpointer.StringPtr(testHostname), }, { - // this endpoint should be ignored for node ports since it's not local + // this endpoint should be ignored for external since it's not local Addresses: []string{"10.0.1.5"}, Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(true), @@ -4668,7 +4668,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { NodeName: utilpointer.StringPtr(testHostname), }, { - // this endpoint should be ignored for node ports since there are ready non-terminating endpoints + // this endpoint should be ignored since it is not ready and the feature gate is off Addresses: []string{"10.0.1.3"}, Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(false), @@ -4678,7 +4678,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { NodeName: utilpointer.StringPtr(testHostname), }, { - // this endpoint should be ignored for node ports since there are ready non-terminating endpoints + // this endpoint should be ignored since it is not ready and the feature gate is off Addresses: []string{"10.0.1.4"}, Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(false), @@ -4688,7 +4688,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { NodeName: utilpointer.StringPtr(testHostname), }, { - // this endpoint should be ignored for node ports since it's not local + // this endpoint should be ignored for external since it's not local Addresses: []string{"10.0.1.5"}, Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(true), @@ -4800,7 +4800,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { NodeName: utilpointer.StringPtr(testHostname), }, { - // this endpoint should be ignored for node ports since it's not local + // this endpoint should be ignored for external since it's not local Addresses: []string{"10.0.1.5"}, Conditions: discovery.EndpointConditions{ Ready: utilpointer.BoolPtr(true), @@ -4861,7 +4861,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { `), }, { - name: "with ProxyTerminatingEndpoints disabled, only terminating endpoints exist", + name: "with ProxyTerminatingEndpoints disabled, only non-local and terminating endpoints exist", line: getLine(), terminatingFeatureGate: false, endpointslice: &discovery.EndpointSlice{ @@ -5961,7 +5961,7 @@ func TestNoEndpointsMetric(t *testing.T) { expectedSyncProxyRulesNoLocalEndpointsTotalExternal int }{ { - name: "internalTrafficPolicy is set and there is non-zero local endpoints", + name: "internalTrafficPolicy is set and there are local endpoints", internalTrafficPolicy: &internalTrafficPolicyLocal, endpoints: []endpoint{ {"10.0.1.1", testHostname}, @@ -5970,7 +5970,7 @@ func TestNoEndpointsMetric(t *testing.T) { }, }, { - name: "externalTrafficPolicy is set and there is non-zero local endpoints", + name: "externalTrafficPolicy is set and there are local endpoints", externalTrafficPolicy: externalTrafficPolicyLocal, endpoints: []endpoint{ {"10.0.1.1", testHostname}, @@ -5979,7 +5979,7 @@ func TestNoEndpointsMetric(t *testing.T) { }, }, { - name: "both policies are set and there is non-zero local endpoints", + name: "both policies are set and there are local endpoints", internalTrafficPolicy: &internalTrafficPolicyLocal, externalTrafficPolicy: externalTrafficPolicyLocal, endpoints: []endpoint{ @@ -5989,7 +5989,7 @@ func TestNoEndpointsMetric(t *testing.T) { }, }, { - name: "internalTrafficPolicy is set and there is zero local endpoint", + name: "internalTrafficPolicy is set and there are no local endpoints", internalTrafficPolicy: &internalTrafficPolicyLocal, endpoints: []endpoint{ {"10.0.1.1", "host0"}, @@ -5999,7 +5999,7 @@ func TestNoEndpointsMetric(t *testing.T) { expectedSyncProxyRulesNoLocalEndpointsTotalInternal: 1, }, { - name: "externalTrafficPolicy is set and there is zero local endpoint", + name: "externalTrafficPolicy is set and there are no local endpoints", externalTrafficPolicy: externalTrafficPolicyLocal, endpoints: []endpoint{ {"10.0.1.1", "host0"}, @@ -6009,7 +6009,7 @@ func TestNoEndpointsMetric(t *testing.T) { expectedSyncProxyRulesNoLocalEndpointsTotalExternal: 1, }, { - name: "both policies are set and there is zero local endpoint", + name: "both policies are set and there are no local endpoints", internalTrafficPolicy: &internalTrafficPolicyLocal, externalTrafficPolicy: externalTrafficPolicyLocal, endpoints: []endpoint{ From cd55f35306909fd7cf6aa4c1c1ec8504d997481b Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 7 Apr 2022 08:31:27 -0400 Subject: [PATCH 3/5] proxy/iptables: tweak sortIPTablesRules a bit It's confusing to have the "this must be the last rule" get sorted to be the first rule in KUBE-SERVICES... --- pkg/proxy/iptables/proxier_test.go | 52 ++++++++++++++++-------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 0e6aeab1b82..e3e25d89728 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -943,10 +943,14 @@ func sortIPTablesRules(ruleData string) (string, error) { lines, output = moveMatchingLines(fmt.Sprintf(`^-A KUBE-NODEPORTS.*%s`, nextNodePortService), lines, output) } - // Move KUBE-SERVICES rules for each service, sorted by service name + // Move KUBE-SERVICES rules for each service, sorted by service name. The + // relative ordering of actual per-service lines doesn't matter, but keep + // the "must be the last rule" rule last because it's confusing otherwise... + lines, tmp := moveMatchingLines(`KUBE-SERVICES.*must be the last rule`, lines, nil) for _, nextService := range findAllMatches(lines, `-A KUBE-SERVICES.*--comment "?([^ ]*)`) { lines, output = moveMatchingLines(fmt.Sprintf(`^-A KUBE-SERVICES.*%s`, nextService), lines, output) } + _, output = moveMatchingLines(`.`, tmp, output) // Move remaining chains, sorted by chain name for _, nextChain := range findAllMatches(lines, `(-A KUBE-[^ ]* )`) { @@ -1080,7 +1084,6 @@ func TestSortIPTablesRules(t *testing.T) { :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3002 -j KUBE-SVC-X27LE4BHSL4DOUIK - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT @@ -1088,6 +1091,7 @@ func TestSortIPTablesRules(t *testing.T) { -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT @@ -1554,7 +1558,6 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT -A KUBE-NODEPORTS -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp --dport 3002 -j KUBE-EXT-PAZTZYUUMV5KCDZL -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3003 -j KUBE-EXT-X27LE4BHSL4DOUIK - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT @@ -1564,6 +1567,7 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 -j KUBE-EXT-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-4SW47YFZTEDKD3PK -m comment --comment "masquerade traffic for ns4/svc4:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-4SW47YFZTEDKD3PK -j KUBE-SVC-4SW47YFZTEDKD3PK -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "pod traffic for ns2/svc2:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT @@ -1728,8 +1732,8 @@ func TestClusterIPEndpointsJump(t *testing.T) { :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -1813,9 +1817,9 @@ func TestLoadBalancer(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-XPGD46QRK7WJZT7O @@ -1896,8 +1900,8 @@ func TestNodePort(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 @@ -2133,9 +2137,9 @@ func TestOnlyLocalExternalIPs(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O @@ -2224,9 +2228,9 @@ func TestNonLocalExternalIPs(t *testing.T) { :KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 192.168.99.11 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 @@ -2446,9 +2450,9 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O @@ -2500,8 +2504,8 @@ func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVL-XPGD46QRK7WJZT7O @@ -2547,8 +2551,8 @@ func TestOnlyLocalNodePorts(t *testing.T) { :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] :KUBE-SVL-XPGD46QRK7WJZT7O - [0:0] -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 192.168.0.2 -j KUBE-NODEPORTS -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "pod traffic for ns1/svc1:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O @@ -3823,8 +3827,8 @@ func TestEndpointSliceE2E(t *testing.T) { :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 0 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4272,8 +4276,8 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4341,8 +4345,8 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4381,8 +4385,8 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { :KUBE-POSTROUTING - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4601,9 +4605,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4721,9 +4725,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4833,9 +4837,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4944,9 +4948,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -5014,9 +5018,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -5260,9 +5264,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 @@ -5371,9 +5375,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 @@ -5475,9 +5479,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 @@ -5636,9 +5640,9 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade traffic for ns1/svc1 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 From f4261283acd2b450694303af5d40752872862fc9 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 10 Mar 2022 12:08:04 -0500 Subject: [PATCH 4/5] proxy/iptables: add packet-flow-based tests of iptables rules Add a new framework for testing out how particular packets would be handled by a given set of iptables rules. (eg, "assert that a packet from 10.180.0.2 to 172.30.0.41:80 gets NATted to 10.180.0.1:80 without being masqueraded"). Add tests using this to all of the existing unit tests. This makes it easier to tell whether a given code change has any effect on behavior, without having to carefully examine the diffs to the generated iptables rules. --- pkg/proxy/iptables/proxier_test.go | 1272 +++++++++++++++++++++++++++- 1 file changed, 1257 insertions(+), 15 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index e3e25d89728..7ccbbff37c1 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -377,6 +377,8 @@ func TestDeleteEndpointConnectionsIPv6(t *testing.T) { const testHostname = "test-hostname" const testNodeIP = "192.168.0.2" +const testExternalClient = "203.0.113.2" +const testExternalClientBlocked = "203.0.113.130" func NewFakeProxier(ipt utiliptables.Interface) *Proxier { // TODO: Call NewProxier after refactoring out the goroutine @@ -1373,6 +1375,591 @@ func assertIPTablesRulesNotEqual(t *testing.T, line int, expected, result string } } +// ruleMatchesIP helps test whether an iptables rule such as "! -s 192.168.0.0/16" matches +// ipStr. ruleAddress is either an IP address ("1.2.3.4") or a CIDR string +// ("1.2.3.0/24"). negated is whether the iptables rule negates the match. +func ruleMatchesIP(t *testing.T, negated bool, ruleAddress, ipStr string) bool { + ip := netutils.ParseIPSloppy(ipStr) + if ip == nil { + t.Fatalf("Bad IP in test case: %s", ipStr) + } + + var matches bool + if strings.Contains(ruleAddress, "/") { + _, cidr, err := netutils.ParseCIDRSloppy(ruleAddress) + if err != nil { + t.Errorf("Bad CIDR in kube-proxy output: %v", err) + } + matches = cidr.Contains(ip) + } else { + ip2 := netutils.ParseIPSloppy(ruleAddress) + if ip2 == nil { + t.Errorf("Bad IP/CIDR in kube-proxy output: %s", ruleAddress) + } + matches = ip.Equal(ip2) + } + return (!negated && matches) || (negated && !matches) +} + +// Regular expressions used by iptablesTracer. Note that these are not fully general-purpose +// and may need to be updated if we make large changes to our iptable rules. +var addRuleToChainRegex = regexp.MustCompile(`-A ([^ ]*) `) +var moduleRegex = regexp.MustCompile("-m ([^ ]*)") +var commentRegex = regexp.MustCompile(`-m comment --comment ("[^"]*"|[^" ]*) `) +var srcLocalRegex = regexp.MustCompile("(!)? --src-type LOCAL") +var destLocalRegex = regexp.MustCompile("(!)? --dst-type LOCAL") +var destIPRegex = regexp.MustCompile("(!)? -d ([^ ]*) ") +var destPortRegex = regexp.MustCompile(" --dport ([^ ]*) ") +var sourceIPRegex = regexp.MustCompile("(!)? -s ([^ ]*) ") +var affinityRegex = regexp.MustCompile(" --rcheck ") + +// (If `--probability` appears, it can only appear before the `-j`, and if `--to-destination` +// appears it can only appear after the `-j`, so this is not as fragile as it looks. +var jumpRegex = regexp.MustCompile("(--probability.*)? -j ([^ ]*)( --to-destination (.*))?$") + +func Test_iptablesTracerRegexps(t *testing.T) { + testCases := []struct { + name string + regex *regexp.Regexp + rule string + matches []string + }{ + { + name: "addRuleToChainRegex", + regex: addRuleToChainRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{`-A KUBE-NODEPORTS `, "KUBE-NODEPORTS"}, + }, + { + name: "addRuleToChainRegex requires an actual rule, not just a chain name", + regex: addRuleToChainRegex, + rule: `-A KUBE-NODEPORTS`, + matches: nil, + }, + { + name: "addRuleToChainRegex only matches adds", + regex: addRuleToChainRegex, + rule: `-D KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: nil, + }, + { + name: "commentRegex with quoted comment", + regex: commentRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{`-m comment --comment "ns2/svc2:p80 health check node port" `, `"ns2/svc2:p80 health check node port"`}, + }, + { + name: "commentRegex with unquoted comment", + regex: commentRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + matches: []string{`-m comment --comment ns1/svc1:p80 `, "ns1/svc1:p80"}, + }, + { + name: "no comment", + regex: commentRegex, + rule: `-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000`, + matches: nil, + }, + { + name: "moduleRegex", + regex: moduleRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{"-m comment", "comment"}, + }, + { + name: "local source", + regex: srcLocalRegex, + rule: `-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ`, + matches: []string{" --src-type LOCAL", ""}, + }, + { + name: "not local destination", + regex: destLocalRegex, + rule: `-A RULE-TYPE-NOT-CURRENTLY-USED-BY-KUBE-PROXY -m addrtype ! --dst-type LOCAL -j KUBE-MARK-MASQ`, + matches: []string{"! --dst-type LOCAL", "!"}, + }, + { + name: "destination IP", + regex: destIPRegex, + rule: `-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`, + matches: []string{" -d 172.30.0.41 ", "", "172.30.0.41"}, + }, + { + name: "destination port", + regex: destPortRegex, + rule: `-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`, + matches: []string{" --dport 80 ", "80"}, + }, + { + name: "destination IP but no port", + regex: destPortRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -d 172.30.0.41 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ`, + matches: nil, + }, + { + name: "source IP", + regex: sourceIPRegex, + rule: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ`, + matches: []string{" -s 10.180.0.1 ", "", "10.180.0.1"}, + }, + { + name: "not source IP", + regex: sourceIPRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ`, + matches: []string{"! -s 10.0.0.0/8 ", "!", "10.0.0.0/8"}, + }, + { + name: "affinityRegex", + regex: affinityRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + matches: []string{" --rcheck "}, + }, + { + name: "jump to internal target", + regex: jumpRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{" -j ACCEPT", "", "ACCEPT", "", ""}, + }, + { + name: "jump to KUBE chain", + regex: jumpRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + matches: []string{" -j KUBE-SEP-SXIVWICOYRO3J4NJ", "", "KUBE-SEP-SXIVWICOYRO3J4NJ", "", ""}, + }, + { + name: "jump to DNAT", + regex: jumpRegex, + rule: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80`, + matches: []string{" -j DNAT --to-destination 10.180.0.1:80", "", "DNAT", " --to-destination 10.180.0.1:80", "10.180.0.1:80"}, + }, + { + name: "jump to endpoint", + regex: jumpRegex, + rule: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + matches: []string{"--probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC", "--probability 0.5000000000", "KUBE-SEP-UKSFD7AGPMPPLUHC", "", ""}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + matches := testCase.regex.FindStringSubmatch(testCase.rule) + if !reflect.DeepEqual(matches, testCase.matches) { + t.Errorf("bad match: expected %#v, got %#v", testCase.matches, matches) + } + }) + } +} + +// knownModules is the set of modules (ie "-m foo") that we allow to be present in rules passed +// to an iptablesTracer. If a rule using another module is found in a rule, the test will +// fail. +// +// If a module is in knownModules but is not in noMatchModules and is not handled by +// ruleMatches, then the result is that match rules using that module will have no effect +// for tracing purposes. +var knownModules = sets.NewString("addrtype", "comment", "conntrack", "mark", "recent", "statistic", "tcp", "udp") + +// noMatchModules is the list of modules that if we see them in a rule, we just +// assume the rule doesn't match and ignore it (because rules with these modules exist +// in the data we are testing against, but aren't relevant to what we're testing). +var noMatchModules = sets.NewString("conntrack", "mark") + +type iptablesChain []string +type iptablesTable map[string]iptablesChain + +// iptablesTracer holds data used while virtually tracing a packet through a set of +// iptables rules +type iptablesTracer struct { + tables map[string]iptablesTable + nodeIP string + t *testing.T + + // matches accumulates the list of rules that were matched, for debugging purposes. + matches []string + + // outputs accumulates the list of matched terminal rule targets (endpoint + // IP:ports, or a special target like "REJECT") and is eventually used to generate + // the return value of tracePacket. + outputs []string + + // markMasq and markDrop track whether the packet has been marked for masquerading + // or dropping. + markMasq bool + markDrop bool +} + +// newIPTablesTracer creates an iptablesTracer. ruleData is an iptables rule dump (as with +// "iptables-save"). nodeIP is the IP to treat as the local node IP (for determining +// whether rules with "--src-type LOCAL" or "--dst-type LOCAL" match). +func newIPTablesTracer(t *testing.T, ruleData, nodeIP string) (*iptablesTracer, error) { + tables, err := parseIPTablesData(ruleData) + if err != nil { + return nil, err + } + + tracer := &iptablesTracer{ + tables: make(map[string]iptablesTable), + nodeIP: nodeIP, + t: t, + } + + for name, rules := range tables { + table := make(iptablesTable) + for _, rule := range rules { + match := addRuleToChainRegex.FindStringSubmatch(rule) + if match != nil { + chainName := match[1] + table[chainName] = append(table[chainName], rule) + } + } + tracer.tables[name] = table + } + + return tracer, nil +} + +// ruleMatches checks if the given iptables rule matches (at least probabilistically) a +// packet with the given sourceIP, destIP, and destPort. (Note that protocol is currently +// ignored.) +func (tracer *iptablesTracer) ruleMatches(rule, sourceIP, destIP, destPort string) bool { + var match []string + + // Delete comments so we don't mistakenly match something in a comment string later + rule = commentRegex.ReplaceAllString(rule, "") + + // Make sure the rule only uses modules ("-m foo") that we are aware of + for _, matches := range moduleRegex.FindAllStringSubmatch(rule, -1) { + moduleName := matches[1] + if !knownModules.Has(moduleName) { + tracer.t.Errorf("Rule %q uses unknown iptables module %q", rule, moduleName) + } + if noMatchModules.Has(moduleName) { + // This rule is doing something irrelevant to iptablesTracer + return false + } + } + + // The sub-rules within an iptables rule are ANDed together, so the rule only + // matches if all of them match. So go through the subrules, and if any of them + // DON'T match, then fail. + + // Match local/non-local. + match = srcLocalRegex.FindStringSubmatch(rule) + if match != nil { + wantLocal := (match[1] != "!") + sourceIsLocal := (sourceIP == tracer.nodeIP || sourceIP == "127.0.0.1") + if wantLocal != sourceIsLocal { + return false + } + } + match = destLocalRegex.FindStringSubmatch(rule) + if match != nil { + wantLocal := (match[1] != "!") + destIsLocal := (destIP == tracer.nodeIP || destIP == "127.0.0.1") + if wantLocal != destIsLocal { + return false + } + } + + // Match destination IP/port. + match = destIPRegex.FindStringSubmatch(rule) + if match != nil { + negated := match[1] == "!" + ruleAddress := match[2] + if !ruleMatchesIP(tracer.t, negated, ruleAddress, destIP) { + return false + } + } + match = destPortRegex.FindStringSubmatch(rule) + if match != nil { + rulePort := match[1] + if rulePort != destPort { + return false + } + } + + // Match source IP (but not currently port) + match = sourceIPRegex.FindStringSubmatch(rule) + if match != nil { + negated := match[1] == "!" + ruleAddress := match[2] + if !ruleMatchesIP(tracer.t, negated, ruleAddress, sourceIP) { + return false + } + } + + // The iptablesTracer has no state/history, so any rule that checks whether affinity + // has been established for a particular endpoint must not match. + if affinityRegex.MatchString(rule) { + return false + } + + // Anything else is assumed to match + return true +} + +// runChain runs the given packet through the rules in the given table and chain, updating +// tracer's internal state accordingly. It returns true if it hits a terminal action. +func (tracer *iptablesTracer) runChain(table, chain, sourceIP, destIP, destPort string) bool { + for _, rule := range tracer.tables[table][chain] { + match := jumpRegex.FindStringSubmatch(rule) + if match == nil { + // You _can_ have rules that don't end in `-j`, but we don't currently + // do that. + tracer.t.Errorf("Could not find jump target in rule %q", rule) + } + isProbabilisticMatch := (match[1] != "") + target := match[2] + natDestination := match[4] + + if !tracer.ruleMatches(rule, sourceIP, destIP, destPort) { + continue + } + // record the matched rule for debugging purposes + tracer.matches = append(tracer.matches, rule) + + switch target { + case "KUBE-MARK-MASQ": + tracer.markMasq = true + continue + + case "KUBE-MARK-DROP": + tracer.markDrop = true + continue + + case "ACCEPT", "REJECT": + // (only valid in filter) + tracer.outputs = append(tracer.outputs, target) + return true + + case "DNAT": + // (only valid in nat) + tracer.outputs = append(tracer.outputs, natDestination) + return true + + default: + // We got a "-j KUBE-SOMETHING", so process that chain + terminated := tracer.runChain(table, target, sourceIP, destIP, destPort) + + // If the subchain hit a terminal rule AND the rule that sent us + // to that chain was non-probabilistic, then this chain terminates + // as well. But if we went there because of a --probability rule, + // then we want to keep accumulating further matches against this + // chain. + if terminated && !isProbabilisticMatch { + return true + } + } + } + + return false +} + +// tracePacket determines what would happen to a packet with the given sourceIP, destIP, +// and destPort, given the indicated iptables ruleData. nodeIP is the local node IP (for +// rules matching "LOCAL"). +// +// The return values are: an array of matched rules (for debugging), the final packet +// destinations (a comma-separated list of IPs, or one of the special targets "ACCEPT", +// "DROP", or "REJECT"), and whether the packet would be masqueraded. +func tracePacket(t *testing.T, ruleData, sourceIP, destIP, destPort, nodeIP string) ([]string, string, bool) { + tracer, err := newIPTablesTracer(t, ruleData, nodeIP) + if err != nil { + t.Errorf("Bad iptables ruleData: %v", err) + } + + // nat:PREROUTING goes first, then the filter chains, then nat:POSTROUTING. For our + // purposes that means we run through the "nat" chains first, starting from the top of + // KUBE-SERVICES, then we do the "filter" chains. The only interesting thing that + // happens in nat:POSTROUTING is that the masquerade mark gets turned into actual + // masquerading. + + // FIXME: we ought to be able to say + // trace.runChain("nat", "PREROUTING", ...) + // here instead of + // trace.runChain("nat", "KUBE-SERVICES", ...) + // (and similarly below with the "filter" chains) but this doesn't work because the + // rules like "-A PREROUTING -j KUBE-SERVICES" are created with iptables.EnsureRule(), + // which iptablestest.FakeIPTables doesn't implement, so those rules will be missing + // from the ruleData we have. So we have to explicitly specify each kube-proxy chain + // we want to run through here. + tracer.runChain("nat", "KUBE-SERVICES", sourceIP, destIP, destPort) + + // Process pending DNAT (which theoretically might affect REJECT/ACCEPT filter rules) + if len(tracer.outputs) != 0 { + destIP = strings.Split(tracer.outputs[0], ":")[0] + } + + // Now run the filter rules to see if the packet is REJECTed or ACCEPTed. The DROP + // rule is created by kubelet, not us, so we have to simulate that manually + if tracer.markDrop { + return tracer.matches, "DROP", false + } + tracer.runChain("filter", "KUBE-SERVICES", sourceIP, destIP, destPort) + tracer.runChain("filter", "KUBE-EXTERNAL-SERVICES", sourceIP, destIP, destPort) + tracer.runChain("filter", "KUBE-NODEPORTS", sourceIP, destIP, destPort) + + return tracer.matches, strings.Join(tracer.outputs, ", "), tracer.markMasq +} + +type packetFlowTest struct { + name string + sourceIP string + destIP string + destPort int + output string + masq bool +} + +func runPacketFlowTests(t *testing.T, line int, ruleData, nodeIP string, testCases []packetFlowTest) { + lineStr := "" + if line != 0 { + lineStr = fmt.Sprintf(" (from line %d)", line) + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + matches, output, masq := tracePacket(t, ruleData, tc.sourceIP, tc.destIP, fmt.Sprintf("%d", tc.destPort), nodeIP) + var errors []string + if output != tc.output { + errors = append(errors, fmt.Sprintf("wrong output: expected %q got %q", tc.output, output)) + } + if masq != tc.masq { + errors = append(errors, fmt.Sprintf("wrong masq: expected %v got %v", tc.masq, masq)) + } + if errors != nil { + t.Errorf("Test %q of a packet from %s to %s:%d%s got result:\n%s\n\nBy matching:\n%s\n\n", + tc.name, tc.sourceIP, tc.destIP, tc.destPort, lineStr, strings.Join(errors, "\n"), strings.Join(matches, "\n")) + } + }) + } +} + +// This tests tracePackets against static data, just to make sure we match things in the +// way we expect to. +func TestTracePackets(t *testing.T) { + rules := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-4SW47YFZTEDKD3PK - [0:0] + :KUBE-EXT-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-EXT-PAZTZYUUMV5KCDZL - [0:0] + :KUBE-EXT-X27LE4BHSL4DOUIK - [0:0] + :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] + :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] + :KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 -j KUBE-EXT-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-EXT-4SW47YFZTEDKD3PK -m comment --comment "masquerade traffic for ns4/svc4:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-4SW47YFZTEDKD3PK -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "pod traffic for ns2/svc2:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.4:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.5:80" -j KUBE-SEP-C6EBXVWJJZMIWKLZ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 -> 10.180.0.2:80" -j KUBE-SEP-RS4RBKLTHTF2IUXJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP + COMMIT + `) + + runPacketFlowTests(t, getLine(), rules, testNodeIP, []packetFlowTest{ + { + name: "no match", + sourceIP: "10.0.0.2", + destIP: "10.0.0.3", + destPort: 80, + output: "", + }, + { + name: "single endpoint", + sourceIP: "10.0.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + }, + { + name: "multiple endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.0.44", + destPort: 80, + output: "10.180.0.4:80, 10.180.0.5:80", + }, + { + name: "LOCAL, KUBE-MARK-MASQ", + sourceIP: testNodeIP, + destIP: "192.168.99.22", + destPort: 80, + output: "10.180.0.2:80", + masq: true, + }, + { + name: "KUBE-MARK-DROP", + sourceIP: testExternalClient, + destIP: "192.168.99.22", + destPort: 80, + output: "DROP", + }, + { + name: "ACCEPT (NodePortHealthCheck)", + sourceIP: testNodeIP, + destIP: testNodeIP, + destPort: 30000, + output: "ACCEPT", + }, + { + name: "REJECT", + sourceIP: "10.0.0.2", + destIP: "172.30.0.43", + destPort: 80, + output: "REJECT", + }, + }) +} + // TestOverallIPTablesRulesWithMultipleServices creates 4 types of services: ClusterIP, // LoadBalancer, ExternalIP and NodePort and verifies if the NAT table rules created // are exactly the same as what is expected. This test provides an overall view of how @@ -1673,6 +2260,16 @@ func TestClusterIPReject(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP rejected", + sourceIP: "10.0.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "REJECT", + }, + }) } func TestClusterIPEndpointsJump(t *testing.T) { @@ -1745,6 +2342,25 @@ func TestClusterIPEndpointsJump(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP accepted", + sourceIP: "10.180.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: false, + }, + { + name: "hairpin to cluster IP", + sourceIP: "10.180.0.1", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + }) } func TestLoadBalancer(t *testing.T) { @@ -1836,6 +2452,62 @@ func TestLoadBalancer(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: false, + }, + { + name: "external to nodePort", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "nodePort bypasses LoadBalancerSourceRanges", + sourceIP: testExternalClientBlocked, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "accepted external to LB", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "blocked external to LB", + sourceIP: testExternalClientBlocked, + destIP: svcLBIP, + destPort: svcPort, + output: "DROP", + }, + { + name: "pod to LB (blocked by LoadBalancerSourceRanges)", + sourceIP: "10.0.0.2", + destIP: svcLBIP, + destPort: svcPort, + output: "DROP", + }, + { + name: "node to LB (blocked by LoadBalancerSourceRanges)", + sourceIP: testNodeIP, + destIP: svcLBIP, + destPort: svcPort, + output: "DROP", + }, + }) } func TestNodePort(t *testing.T) { @@ -1915,6 +2587,41 @@ func TestNodePort(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: false, + }, + { + name: "external to nodePort", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "node to nodePort", + sourceIP: testNodeIP, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "localhost to nodePort gets masqueraded", + sourceIP: "127.0.0.1", + destIP: "127.0.0.1", + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + }) } func TestHealthCheckNodePort(t *testing.T) { @@ -1932,21 +2639,19 @@ func TestHealthCheckNodePort(t *testing.T) { Protocol: v1.ProtocolTCP, } - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "LoadBalancer" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.HealthCheckNodePort = int32(svcHealthCheckNodePort) - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal - }), - ) - + svc := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + svc.Spec.HealthCheckNodePort = int32(svcHealthCheckNodePort) + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + }) + makeServiceMap(fp, svc) fp.syncProxyRules() expected := dedent.Dedent(` @@ -1976,6 +2681,30 @@ func TestHealthCheckNodePort(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "firewall accepts HealthCheckNodePort", + sourceIP: "1.2.3.4", + destIP: testNodeIP, + destPort: svcHealthCheckNodePort, + output: "ACCEPT", + masq: false, + }, + }) + + fp.OnServiceDelete(svc) + fp.syncProxyRules() + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "HealthCheckNodePort no longer has any rule", + sourceIP: "1.2.3.4", + destIP: testNodeIP, + destPort: svcHealthCheckNodePort, + output: "", + }, + }) } func TestMasqueradeRule(t *testing.T) { @@ -2068,6 +2797,23 @@ func TestExternalIPsReject(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP with no endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "external IP with no endpoints", + sourceIP: testExternalClient, + destIP: svcExternalIPs, + destPort: svcPort, + output: "REJECT", + }, + }) } func TestOnlyLocalExternalIPs(t *testing.T) { @@ -2159,6 +2905,25 @@ func TestOnlyLocalExternalIPs(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP hits both endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external IP hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: svcExternalIPs, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + }) } // TestNonLocalExternalIPs tests if we add the masquerade rule into svcChain in order to @@ -2247,6 +3012,25 @@ func TestNonLocalExternalIPs(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to external IP", + sourceIP: testExternalClient, + destIP: svcExternalIPs, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: true, + }, + }) } func TestNodePortReject(t *testing.T) { @@ -2300,6 +3084,30 @@ func TestNodePortReject(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "pod to NodePort", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: "REJECT", + }, + { + name: "external to NodePort", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: "REJECT", + }, + }) } func TestLoadBalancerReject(t *testing.T) { @@ -2367,6 +3175,30 @@ func TestLoadBalancerReject(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "pod to LoadBalancer IP", + sourceIP: "10.0.0.2", + destIP: svcLBIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "external to LoadBalancer IP", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: "REJECT", + }, + }) } func TestOnlyLocalLoadBalancing(t *testing.T) { @@ -2475,6 +3307,33 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP hits both endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to LB IP hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + { + name: "external to NodePort hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + }) } func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { @@ -2622,6 +3481,59 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable fp.syncProxyRules() assertIPTablesRulesEqual(t, line, expected, fp.iptablesData.String()) + + runPacketFlowTests(t, line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP hit both endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to NodePort hits only local endpoint", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + { + name: "pod to localhost doesn't work because localhost is not in nodePortAddresses", + sourceIP: "10.0.0.2", + destIP: "127.0.0.1", + destPort: svcNodePort, + output: "", + }, + }) + + if fp.localDetector.IsImplemented() { + // pod-to-NodePort is treated as internal traffic, so we see both endpoints + runPacketFlowTests(t, line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to NodePort hits both endpoints", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + }) + } else { + // pod-to-NodePort is (incorrectly) treated as external traffic + // when there is no LocalTrafficDetector. + runPacketFlowTests(t, line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to NodePort hits only local endpoint", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + }) + } } func TestComputeProbability(t *testing.T) { @@ -4303,6 +5215,7 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { endpoints []endpoint expectEndpointRule bool expectedIPTablesWithSlice string + flowTests []packetFlowTest }{ { name: "internalTrafficPolicy is cluster", @@ -4316,6 +5229,16 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { }, expectEndpointRule: true, expectedIPTablesWithSlice: clusterExpectedIPTables, + flowTests: []packetFlowTest{ + { + name: "pod to ClusterIP hits all endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.3:80", + masq: false, + }, + }, }, { name: "internalTrafficPolicy is local and there are local endpoints", @@ -4357,6 +5280,16 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -j KUBE-SEP-3JOIVZTXZZRGORX4 COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to ClusterIP hits only local endpoint", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80", + masq: false, + }, + }, }, { name: "internalTrafficPolicy is local and there are no local endpoints", @@ -4395,6 +5328,15 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), + flowTests: []packetFlowTest{ + { + name: "no endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "DROP", + }, + }, }, { name: "Local internalTrafficPolicy is ignored when feature gate is off", @@ -4408,6 +5350,16 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { }, expectEndpointRule: false, expectedIPTablesWithSlice: clusterExpectedIPTables, + flowTests: []packetFlowTest{ + { + name: "pod to ClusterIP hits all endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.3:80", + masq: false, + }, + }, }, } @@ -4461,12 +5413,24 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() assertIPTablesRulesEqual(t, tc.line, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) + runPacketFlowTests(t, tc.line, fp.iptablesData.String(), testNodeIP, tc.flowTests) + fp.OnEndpointSliceDelete(endpointSlice) + fp.syncProxyRules() if tc.expectEndpointRule { fp.OnEndpointSliceDelete(endpointSlice) fp.syncProxyRules() assertIPTablesRulesNotEqual(t, tc.line, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) } + runPacketFlowTests(t, tc.line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "endpoints deleted", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + }) }) } } @@ -4515,6 +5479,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { endpointslice *discovery.EndpointSlice expectedIPTables string noUsableEndpoints bool + flowTests []packetFlowTest }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", @@ -4635,6 +5600,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -j KUBE-SEP-IO5XOSKPAXIFQXAJ COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80", + masq: false, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist", @@ -4755,6 +5738,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -j KUBE-SEP-IO5XOSKPAXIFQXAJ COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80", + masq: false, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist", @@ -4863,6 +5864,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.2:80, 10.0.1.3:80", + masq: false, + }, + }, }, { name: "with ProxyTerminatingEndpoints disabled, only non-local and terminating endpoints exist", @@ -4967,6 +5986,23 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "DROP", + }, + }, }, { name: "ProxyTerminatingEndpoints enabled, terminating endpoints on remote node", @@ -5037,6 +6073,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + }, + { + name: "external to LB, no locally-usable endpoints", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "DROP", + }, + }, }, { name: "no usable endpoints on any node", @@ -5103,6 +6155,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP, no usable endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB, no usable endpoints", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }, }, } @@ -5120,6 +6188,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() @@ -5129,6 +6198,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { } else { assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to clusterIP after endpoints deleted", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB after endpoints deleted", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }) }) } } @@ -5177,6 +6262,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) endpointslice *discovery.EndpointSlice expectedIPTables string noUsableEndpoints bool + flowTests []packetFlowTest }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", @@ -5288,6 +6374,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: true, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist", @@ -5399,6 +6503,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: true, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist", @@ -5503,6 +6625,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.2:80, 10.0.1.3:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.2:80, 10.0.1.3:80, 10.0.1.5:80", + masq: true, + }, + }, }, { name: "with ProxyTerminatingEndpoints disabled, only terminating endpoints exist", @@ -5593,6 +6733,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }, }, { name: "ProxyTerminatingEndpoints enabled, terminating endpoints on remote node", @@ -5656,6 +6812,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.5:80", + masq: true, + }, + }, }, { name: "no usable endpoints on any node", @@ -5721,6 +6895,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }, }, } @@ -5738,6 +6928,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() @@ -5747,6 +6938,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) } else { assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to clusterIP after endpoints deleted", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB after endpoints deleted", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }) }) } } @@ -5824,6 +7031,41 @@ func TestMasqueradeAll(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + { + name: "pod to LB", + sourceIP: "10.0.0.2", + destIP: "1.2.3.4", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + { + name: "external to cluster IP", + sourceIP: testExternalClient, + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + }) } func countEndpointsAndComments(iptablesData string, matchEndpoint string) (string, int, int) { From 0ecf11a23b929967edb2205cc96d1bfffa77db8f Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Mon, 10 Jan 2022 14:07:16 -0500 Subject: [PATCH 5/5] proxy/iptables: add a general test of internal vs external traffic Add TestInternalExternalMasquerade, which tests whether various packets are considered internal or external for purposes of traffic policy, and whether they get masqueraded, with and without --masquerade-all, with and without a working LocalTrafficDetector. (This extends and replaces the old TestMasqueradeAll.) --- pkg/proxy/iptables/proxier_test.go | 616 +++++++++++++++++++++++++---- 1 file changed, 538 insertions(+), 78 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 7ccbbff37c1..fc2feadd874 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -6958,87 +6958,137 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) } } -func TestMasqueradeAll(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.masqueradeAll = true +func TestInternalExternalMasquerade(t *testing.T) { + // (Put the test setup code in an internal function so we can have it here at the + // top, before the test cases that will be run against it.) + setupTest := func(fp *Proxier) { + local := v1.ServiceInternalTrafficPolicyLocal + tcpProtocol := v1.ProtocolTCP - makeServiceMap(fp, - makeTestService("ns1", "svc1", func(svc *v1.Service) { - svc.Spec.Type = "LoadBalancer" - svc.Spec.ClusterIP = "172.30.0.41" - svc.Spec.Ports = []v1.ServicePort{{ - Name: "p80", - Port: 80, - Protocol: v1.ProtocolTCP, - NodePort: int32(3001), - }} - svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ - IP: "1.2.3.4", - }} - }), - ) + makeServiceMap(fp, + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = "172.30.0.41" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: int32(3001), + }} + svc.Spec.HealthCheckNodePort = 30001 + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: "1.2.3.4", + }} + }), + makeTestService("ns2", "svc2", func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = "172.30.0.42" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: int32(3002), + }} + svc.Spec.HealthCheckNodePort = 30002 + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: "5.6.7.8", + }} + }), + makeTestService("ns3", "svc3", func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = "172.30.0.43" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: int32(3003), + }} + svc.Spec.HealthCheckNodePort = 30003 + svc.Spec.InternalTrafficPolicy = &local + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: "9.10.11.12", + }} + }), + ) - tcpProtocol := v1.ProtocolTCP - populateEndpointSlices(fp, - makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{"10.180.0.1"}, - }} - eps.Ports = []discovery.EndpointPort{{ - Name: utilpointer.StringPtr("p80"), - Port: utilpointer.Int32(80), - Protocol: &tcpProtocol, - }} - }), - ) + populateEndpointSlices(fp, + makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { + eps.AddressType = discovery.AddressTypeIPv4 + eps.Endpoints = []discovery.Endpoint{ + { + Addresses: []string{"10.180.0.1"}, + NodeName: utilpointer.StringPtr(testHostname), + }, + { + Addresses: []string{"10.180.1.1"}, + NodeName: utilpointer.StringPtr("remote"), + }, + } + eps.Ports = []discovery.EndpointPort{{ + Name: utilpointer.StringPtr("p80"), + Port: utilpointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), + makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) { + eps.AddressType = discovery.AddressTypeIPv4 + eps.Endpoints = []discovery.Endpoint{ + { + Addresses: []string{"10.180.0.2"}, + NodeName: utilpointer.StringPtr(testHostname), + }, + { + Addresses: []string{"10.180.1.2"}, + NodeName: utilpointer.StringPtr("remote"), + }, + } + eps.Ports = []discovery.EndpointPort{{ + Name: utilpointer.StringPtr("p80"), + Port: utilpointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), + makeTestEndpointSlice("ns3", "svc3", 1, func(eps *discovery.EndpointSlice) { + eps.AddressType = discovery.AddressTypeIPv4 + eps.Endpoints = []discovery.Endpoint{ + { + Addresses: []string{"10.180.0.3"}, + NodeName: utilpointer.StringPtr(testHostname), + }, + { + Addresses: []string{"10.180.1.3"}, + NodeName: utilpointer.StringPtr("remote"), + }, + } + eps.Ports = []discovery.EndpointPort{{ + Name: utilpointer.StringPtr("p80"), + Port: utilpointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), + ) - fp.syncProxyRules() + fp.syncProxyRules() + } - expected := dedent.Dedent(` - *filter - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - :KUBE-SERVICES - [0:0] - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-EXT-XPGD46QRK7WJZT7O - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-NODEPORTS - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - :KUBE-SERVICES - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-XPGD46QRK7WJZT7O - -A KUBE-EXT-XPGD46QRK7WJZT7O -m comment --comment "masquerade traffic for ns1/svc1:p80 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-XPGD46QRK7WJZT7O -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ - COMMIT - `) - assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) - - runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + // We use the same flowTests for all of the testCases. The "output" and "masq" + // values here represent the normal case (working localDetector, no masqueradeAll) + flowTests := []packetFlowTest{ { - name: "pod to cluster IP", + name: "pod to ClusterIP", sourceIP: "10.0.0.2", destIP: "172.30.0.41", destPort: 80, - output: "10.180.0.1:80", + output: "10.180.0.1:80, 10.180.1.1:80", + masq: false, + }, + { + name: "pod to NodePort", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: 3001, + output: "10.180.0.1:80, 10.180.1.1:80", masq: true, }, { @@ -7046,15 +7096,55 @@ func TestMasqueradeAll(t *testing.T) { sourceIP: "10.0.0.2", destIP: "1.2.3.4", destPort: 80, - output: "10.180.0.1:80", + output: "10.180.0.1:80, 10.180.1.1:80", masq: true, }, { - name: "external to cluster IP", + name: "node to ClusterIP", + sourceIP: testNodeIP, + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80, 10.180.1.1:80", + masq: true, + }, + { + name: "node to NodePort", + sourceIP: testNodeIP, + destIP: testNodeIP, + destPort: 3001, + output: "10.180.0.1:80, 10.180.1.1:80", + masq: true, + }, + { + name: "localhost to NodePort", + sourceIP: "127.0.0.1", + destIP: "127.0.0.1", + destPort: 3001, + output: "10.180.0.1:80, 10.180.1.1:80", + masq: true, + }, + { + name: "node to LB", + sourceIP: testNodeIP, + destIP: "1.2.3.4", + destPort: 80, + output: "10.180.0.1:80, 10.180.1.1:80", + masq: true, + }, + { + name: "external to ClusterIP", sourceIP: testExternalClient, destIP: "172.30.0.41", destPort: 80, - output: "10.180.0.1:80", + output: "10.180.0.1:80, 10.180.1.1:80", + masq: true, + }, + { + name: "external to NodePort", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: 3001, + output: "10.180.0.1:80, 10.180.1.1:80", masq: true, }, { @@ -7062,10 +7152,380 @@ func TestMasqueradeAll(t *testing.T) { sourceIP: testExternalClient, destIP: "1.2.3.4", destPort: 80, - output: "10.180.0.1:80", + output: "10.180.0.1:80, 10.180.1.1:80", masq: true, }, - }) + { + name: "pod to ClusterIP with eTP:Local", + sourceIP: "10.0.0.2", + destIP: "172.30.0.42", + destPort: 80, + + // externalTrafficPolicy does not apply to ClusterIP traffic, so same + // as "Pod to ClusterIP" + output: "10.180.0.2:80, 10.180.1.2:80", + masq: false, + }, + { + name: "pod to NodePort with eTP:Local", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: 3002, + + // FIXME: The short-circuit rule means we potentially send to a remote + // endpoint without masquerading, which is inconsistent with the + // eTP:Cluster case. We should either be masquerading here, or NOT + // masquerading in the "pod to NodePort" case above. + output: "10.180.0.2:80, 10.180.1.2:80", + masq: false, + }, + { + name: "pod to LB with eTP:Local", + sourceIP: "10.0.0.2", + destIP: "5.6.7.8", + destPort: 80, + + // FIXME: The short-circuit rule means we potentially send to a remote + // endpoint without masquerading, which is inconsistent with the + // eTP:Cluster case. We should either be masquerading here, or NOT + // masquerading in the "pod to LB" case above. + output: "10.180.0.2:80, 10.180.1.2:80", + masq: false, + }, + { + name: "node to ClusterIP with eTP:Local", + sourceIP: testNodeIP, + destIP: "172.30.0.42", + destPort: 80, + + // externalTrafficPolicy does not apply to ClusterIP traffic, so same + // as "node to ClusterIP" + output: "10.180.0.2:80, 10.180.1.2:80", + masq: true, + }, + { + name: "node to NodePort with eTP:Local", + sourceIP: testNodeIP, + destIP: testNodeIP, + destPort: 3001, + + // The traffic gets short-circuited, ignoring externalTrafficPolicy, so + // same as "node to NodePort" above. + output: "10.180.0.1:80, 10.180.1.1:80", + masq: true, + }, + { + name: "localhost to NodePort with eTP:Local", + sourceIP: "127.0.0.1", + destIP: "127.0.0.1", + destPort: 3002, + + // The traffic gets short-circuited, ignoring externalTrafficPolicy, so + // same as "localhost to NodePort" above. + output: "10.180.0.2:80, 10.180.1.2:80", + masq: true, + }, + { + name: "node to LB with eTP:Local", + sourceIP: testNodeIP, + destIP: "5.6.7.8", + destPort: 80, + + // The traffic gets short-circuited, ignoring externalTrafficPolicy, so + // same as "node to LB" above. + output: "10.180.0.2:80, 10.180.1.2:80", + masq: true, + }, + { + name: "external to ClusterIP with eTP:Local", + sourceIP: testExternalClient, + destIP: "172.30.0.42", + destPort: 80, + + // externalTrafficPolicy does not apply to ClusterIP traffic, so same + // as "external to ClusterIP" above. + output: "10.180.0.2:80, 10.180.1.2:80", + masq: true, + }, + { + name: "external to NodePort with eTP:Local", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: 3002, + + // externalTrafficPolicy applies; only the local endpoint is + // selected, and we don't masquerade. + output: "10.180.0.2:80", + masq: false, + }, + { + name: "external to LB with eTP:Local", + sourceIP: testExternalClient, + destIP: "5.6.7.8", + destPort: 80, + + // externalTrafficPolicy applies; only the local endpoint is + // selected, and we don't masquerade. + output: "10.180.0.2:80", + masq: false, + }, + { + name: "pod to ClusterIP with iTP:Local", + sourceIP: "10.0.0.2", + destIP: "172.30.0.43", + destPort: 80, + + // internalTrafficPolicy applies; only the local endpoint is + // selected. + output: "10.180.0.3:80", + masq: false, + }, + { + name: "pod to NodePort with iTP:Local", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: 3003, + + // internalTrafficPolicy does not apply to NodePort traffic, so same as + // "pod to NodePort" above. + output: "10.180.0.3:80, 10.180.1.3:80", + masq: true, + }, + { + name: "pod to LB with iTP:Local", + sourceIP: "10.0.0.2", + destIP: "9.10.11.12", + destPort: 80, + + // internalTrafficPolicy does not apply to LoadBalancer traffic, so + // same as "pod to LB" above. + output: "10.180.0.3:80, 10.180.1.3:80", + masq: true, + }, + { + name: "node to ClusterIP with iTP:Local", + sourceIP: testNodeIP, + destIP: "172.30.0.43", + destPort: 80, + + // internalTrafficPolicy applies; only the local endpoint is selected. + // Traffic is masqueraded as in the "node to ClusterIP" case because + // internalTrafficPolicy does not affect masquerading. + output: "10.180.0.3:80", + masq: true, + }, + { + name: "node to NodePort with iTP:Local", + sourceIP: testNodeIP, + destIP: testNodeIP, + destPort: 3003, + + // internalTrafficPolicy does not apply to NodePort traffic, so same as + // "node to NodePort" above. + output: "10.180.0.3:80, 10.180.1.3:80", + masq: true, + }, + { + name: "localhost to NodePort with iTP:Local", + sourceIP: "127.0.0.1", + destIP: "127.0.0.1", + destPort: 3003, + + // internalTrafficPolicy does not apply to NodePort traffic, so same as + // "localhost to NodePort" above. + output: "10.180.0.3:80, 10.180.1.3:80", + masq: true, + }, + { + name: "node to LB with iTP:Local", + sourceIP: testNodeIP, + destIP: "9.10.11.12", + destPort: 80, + + // internalTrafficPolicy does not apply to LoadBalancer traffic, so + // same as "node to LB" above. + output: "10.180.0.3:80, 10.180.1.3:80", + masq: true, + }, + { + name: "external to ClusterIP with iTP:Local", + sourceIP: testExternalClient, + destIP: "172.30.0.43", + destPort: 80, + + // internalTrafficPolicy applies; only the local endpoint is selected. + // Traffic is masqueraded as in the "external to ClusterIP" case + // because internalTrafficPolicy does not affect masquerading. + output: "10.180.0.3:80", + masq: true, + }, + { + name: "external to NodePort with iTP:Local", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: 3003, + + // internalTrafficPolicy does not apply to NodePort traffic, so same as + // "external to NodePort" above. + output: "10.180.0.3:80, 10.180.1.3:80", + masq: true, + }, + { + name: "external to LB with iTP:Local", + sourceIP: testExternalClient, + destIP: "9.10.11.12", + destPort: 80, + + // internalTrafficPolicy does not apply to LoadBalancer traffic, so + // same as "external to LB" above. + output: "10.180.0.3:80, 10.180.1.3:80", + masq: true, + }, + } + + type packetFlowTestOverride struct { + output *string + masq *bool + } + + testCases := []struct { + name string + line int + masqueradeAll bool + localDetector bool + overrides map[string]packetFlowTestOverride + }{ + { + name: "base", + line: getLine(), + masqueradeAll: false, + localDetector: true, + overrides: nil, + }, + { + name: "no LocalTrafficDetector", + line: getLine(), + masqueradeAll: false, + localDetector: false, + overrides: map[string]packetFlowTestOverride{ + // With no LocalTrafficDetector, all traffic to a + // ClusterIP is assumed to be from a pod, and thus to not + // require masquerading. + "node to ClusterIP": { + masq: utilpointer.Bool(false), + }, + "node to ClusterIP with eTP:Local": { + masq: utilpointer.Bool(false), + }, + "node to ClusterIP with iTP:Local": { + masq: utilpointer.Bool(false), + }, + "external to ClusterIP": { + masq: utilpointer.Bool(false), + }, + "external to ClusterIP with eTP:Local": { + masq: utilpointer.Bool(false), + }, + "external to ClusterIP with iTP:Local": { + masq: utilpointer.Bool(false), + }, + + // And there's no eTP:Local short-circuit for pod traffic, + // so pods get only the local endpoints. + "pod to NodePort with eTP:Local": { + output: utilpointer.String("10.180.0.2:80"), + }, + "pod to LB with eTP:Local": { + output: utilpointer.String("10.180.0.2:80"), + }, + }, + }, + { + name: "masqueradeAll", + line: getLine(), + masqueradeAll: true, + localDetector: true, + overrides: map[string]packetFlowTestOverride{ + // All "to ClusterIP" traffic gets masqueraded when using + // --masquerade-all. + "pod to ClusterIP": { + masq: utilpointer.Bool(true), + }, + "pod to ClusterIP with eTP:Local": { + masq: utilpointer.Bool(true), + }, + "pod to ClusterIP with iTP:Local": { + masq: utilpointer.Bool(true), + }, + }, + }, + { + name: "masqueradeAll, no LocalTrafficDetector", + line: getLine(), + masqueradeAll: true, + localDetector: false, + overrides: map[string]packetFlowTestOverride{ + // As in "masqueradeAll" + "pod to ClusterIP": { + masq: utilpointer.Bool(true), + }, + "pod to ClusterIP with eTP:Local": { + masq: utilpointer.Bool(true), + }, + "pod to ClusterIP with iTP:Local": { + masq: utilpointer.Bool(true), + }, + + // As in "no LocalTrafficDetector" + "pod to NodePort with eTP:Local": { + output: utilpointer.String("10.180.0.2:80"), + }, + "pod to LB with eTP:Local": { + output: utilpointer.String("10.180.0.2:80"), + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, true)() + + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + fp.masqueradeAll = tc.masqueradeAll + if !tc.localDetector { + fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() + } + setupTest(fp) + + // Merge base flowTests with per-test-case overrides + tcFlowTests := make([]packetFlowTest, len(flowTests)) + overridesApplied := 0 + for i := range flowTests { + tcFlowTests[i] = flowTests[i] + if overrides, set := tc.overrides[flowTests[i].name]; set { + overridesApplied++ + if overrides.masq != nil { + if tcFlowTests[i].masq == *overrides.masq { + t.Errorf("%q override value for masq is same as base value", flowTests[i].name) + } + tcFlowTests[i].masq = *overrides.masq + } + if overrides.output != nil { + if tcFlowTests[i].output == *overrides.output { + t.Errorf("%q override value for output is same as base value", flowTests[i].name) + } + tcFlowTests[i].output = *overrides.output + } + } + } + if overridesApplied != len(tc.overrides) { + t.Errorf("%d overrides did not match any test case name!", len(tc.overrides)-overridesApplied) + } + runPacketFlowTests(t, tc.line, fp.iptablesData.String(), testNodeIP, tcFlowTests) + }) + } } func countEndpointsAndComments(iptablesData string, matchEndpoint string) (string, int, int) {