From 59b7f969e83b8e3471f4d113a1ce757f97dcee01 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 29 Jun 2022 13:35:37 -0400 Subject: [PATCH 1/3] proxy/iptables: fix up TestOverallIPTablesRulesWithMultipleServices svc2b was using the same ClusterIP as svc3; change it and rename the service to svc5 to make everything clearer. Move the test of LoadBalancerSourceRanges from svc2 to svc5, so that svc2 tests the rules for dropping packets due to externalTrafficPolicy, and svc5 tests the rules for dropping packets due to LoadBalancerSourceRanges, rather than having them both mixed together in svc2. Add svc6 with no endpoints. --- pkg/proxy/iptables/proxier_test.go | 108 ++++++++++++++++------------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 6a885aabd54..2c46fd744be 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1791,7 +1791,7 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { Protocol: v1.ProtocolTCP, }} }), - // create LoadBalancer service + // create LoadBalancer service with Local traffic policy makeTestService("ns2", "svc2", func(svc *v1.Service) { svc.Spec.Type = "LoadBalancer" svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal @@ -1805,26 +1805,7 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ IP: "1.2.3.4", }} - // Also ensure that invalid LoadBalancerSourceRanges will not result - // in a crash. svc.Spec.ExternalIPs = []string{"192.168.99.22"} - svc.Spec.LoadBalancerSourceRanges = []string{" 203.0.113.0/25"} - svc.Spec.HealthCheckNodePort = 30000 - }), - // create LoadBalancer service with Cluster traffic policy and no source ranges - makeTestService("ns2b", "svc2b", func(svc *v1.Service) { - svc.Spec.Type = "LoadBalancer" - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster - svc.Spec.ClusterIP = "172.30.0.43" - svc.Spec.Ports = []v1.ServicePort{{ - Name: "p80", - Port: 80, - Protocol: v1.ProtocolTCP, - NodePort: 3002, - }} - svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ - IP: "5.6.7.8", - }} svc.Spec.HealthCheckNodePort = 30000 }), // create NodePort service @@ -1850,6 +1831,36 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { TargetPort: intstr.FromInt(80), }} }), + // create LoadBalancer service with Cluster traffic policy and source ranges + makeTestService("ns5", "svc5", func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster + svc.Spec.ClusterIP = "172.30.0.45" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: 3002, + }} + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: "5.6.7.8", + }} + svc.Spec.HealthCheckNodePort = 30000 + // Extra whitespace to ensure that invalid value will not result + // in a crash, for backward compatibility. + svc.Spec.LoadBalancerSourceRanges = []string{" 203.0.113.0/25"} + }), + // create ClusterIP service with no endpoints + makeTestService("ns6", "svc6", func(svc *v1.Service) { + svc.Spec.Type = "ClusterIP" + svc.Spec.ClusterIP = "172.30.0.46" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + TargetPort: intstr.FromInt(80), + }} + }), ) populateEndpointSlices(fp, // create ClusterIP service endpoints @@ -1877,18 +1888,6 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { Protocol: &tcpProtocol, }} }), - // create Cluster LoadBalancer endpoints - makeTestEndpointSlice("ns2b", "svc2b", 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{"10.180.0.3"}, - }} - eps.Ports = []discovery.EndpointPort{{ - Name: utilpointer.StringPtr("p80"), - Port: utilpointer.Int32(80), - Protocol: &tcpProtocol, - }} - }), // create NodePort service endpoints makeTestEndpointSlice("ns3", "svc3", 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 @@ -1916,6 +1915,18 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { Protocol: &tcpProtocol, }} }), + // create Cluster LoadBalancer endpoints + makeTestEndpointSlice("ns5", "svc5", 1, func(eps *discovery.EndpointSlice) { + eps.AddressType = discovery.AddressTypeIPv4 + eps.Endpoints = []discovery.Endpoint{{ + Addresses: []string{"10.180.0.3"}, + }} + eps.Ports = []discovery.EndpointPort{{ + Name: utilpointer.StringPtr("p80"), + Port: utilpointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), ) fp.syncProxyRules() @@ -1927,6 +1938,7 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-SERVICES -m comment --comment "ns6/svc6:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.46 --dport 80 -j REJECT -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -1936,35 +1948,35 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-EXT-4SW47YFZTEDKD3PK - [0:0] :KUBE-EXT-GNZBNJ2PO5MGZ6GT - [0:0] - :KUBE-EXT-PAZTZYUUMV5KCDZL - [0:0] + :KUBE-EXT-NUKIZ6OKUXPJNT4C - [0:0] :KUBE-EXT-X27LE4BHSL4DOUIK - [0:0] - :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-FW-NUKIZ6OKUXPJNT4C - [0:0] :KUBE-MARK-MASQ - [0:0] :KUBE-POSTROUTING - [0:0] :KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] + :KUBE-SEP-I77PXRDZVX7PMWMN - [0:0] :KUBE-SEP-OYPFS5VJICHGATKP - [0:0] - :KUBE-SEP-QDCEFMBQEGWIV4VT - [0:0] :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] :KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] :KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] - :KUBE-SVC-PAZTZYUUMV5KCDZL - [0:0] + :KUBE-SVC-NUKIZ6OKUXPJNT4C - [0:0] :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT - -A KUBE-NODEPORTS -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp --dport 3002 -j KUBE-EXT-PAZTZYUUMV5KCDZL -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3003 -j KUBE-EXT-X27LE4BHSL4DOUIK + -A KUBE-NODEPORTS -m comment --comment ns5/svc5:p80 -m tcp -p tcp --dport 3002 -j KUBE-EXT-NUKIZ6OKUXPJNT4C -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT - -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT - -A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-PAZTZYUUMV5KCDZL - -A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -m tcp -p tcp -d 5.6.7.8 --dport 80 -j KUBE-EXT-PAZTZYUUMV5KCDZL + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 -j KUBE-EXT-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "ns5/svc5:p80 cluster IP" -m tcp -p tcp -d 172.30.0.45 --dport 80 -j KUBE-SVC-NUKIZ6OKUXPJNT4C + -A KUBE-SERVICES -m comment --comment "ns5/svc5:p80 loadbalancer IP" -m tcp -p tcp -d 5.6.7.8 --dport 80 -j KUBE-FW-NUKIZ6OKUXPJNT4C -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-4SW47YFZTEDKD3PK -m comment --comment "masquerade traffic for ns4/svc4:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-4SW47YFZTEDKD3PK -j KUBE-SVC-4SW47YFZTEDKD3PK @@ -1972,22 +1984,22 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT - -A KUBE-EXT-PAZTZYUUMV5KCDZL -m comment --comment "masquerade traffic for ns2b/svc2b:p80 external destinations" -j KUBE-MARK-MASQ - -A KUBE-EXT-PAZTZYUUMV5KCDZL -j KUBE-SVC-PAZTZYUUMV5KCDZL + -A KUBE-EXT-NUKIZ6OKUXPJNT4C -m comment --comment "masquerade traffic for ns5/svc5:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-NUKIZ6OKUXPJNT4C -j KUBE-SVC-NUKIZ6OKUXPJNT4C -A KUBE-EXT-X27LE4BHSL4DOUIK -m comment --comment "masquerade traffic for ns3/svc3:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-X27LE4BHSL4DOUIK -j KUBE-SVC-X27LE4BHSL4DOUIK - -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT - -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP + -A KUBE-FW-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-NUKIZ6OKUXPJNT4C + -A KUBE-FW-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 loadbalancer IP" -j KUBE-MARK-DROP -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 + -A KUBE-SEP-I77PXRDZVX7PMWMN -m comment --comment ns5/svc5:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-I77PXRDZVX7PMWMN -m comment --comment ns5/svc5:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 - -A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ - -A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ @@ -1999,8 +2011,8 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.5:80" -j KUBE-SEP-C6EBXVWJJZMIWKLZ -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 -> 10.180.0.2:80" -j KUBE-SEP-RS4RBKLTHTF2IUXJ - -A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 -> 10.180.0.3:80" -j KUBE-SEP-QDCEFMBQEGWIV4VT + -A KUBE-SVC-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 cluster IP" -m tcp -p tcp -d 172.30.0.45 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 -> 10.180.0.3:80" -j KUBE-SEP-I77PXRDZVX7PMWMN -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 -> 10.180.0.3:80" -j KUBE-SEP-OYPFS5VJICHGATKP -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ From 283218bd4c8d3aa74493b4d0fecf35be1a391c90 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 13 Apr 2022 13:08:21 -0400 Subject: [PATCH 2/3] proxy/iptables: update TestTracePackets Re-sync the rules from TestOverallIPTablesRulesWithMultipleServices to make sure we're testing all the right kinds of rules. Remove a duplicate copy of the KUBE-MARK-MASQ and KUBE-POSTROUTING rules. Update the "REJECT" test to use the new svc6 from TestOverallIPTablesRulesWithMultipleServices. (Previously it had used a modified version of TOIPTRWMS's svc3.) --- pkg/proxy/iptables/proxier_test.go | 39 +++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 2c46fd744be..6ea64f49900 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1638,7 +1638,7 @@ func TestTracePackets(t *testing.T) { -A FORWARD -m conntrack --ctstate NEW -m comment --comment kubernetes externally-visible service portals -j KUBE-EXTERNAL-SERVICES -A OUTPUT -m conntrack --ctstate NEW -m comment --comment kubernetes service portals -j KUBE-SERVICES -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j REJECT + -A KUBE-SERVICES -m comment --comment "ns6/svc6:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.46 --dport 80 -j REJECT -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -1650,19 +1650,23 @@ func TestTracePackets(t *testing.T) { :POSTROUTING - [0:0] :KUBE-EXT-4SW47YFZTEDKD3PK - [0:0] :KUBE-EXT-GNZBNJ2PO5MGZ6GT - [0:0] - :KUBE-EXT-PAZTZYUUMV5KCDZL - [0:0] + :KUBE-EXT-NUKIZ6OKUXPJNT4C - [0:0] :KUBE-EXT-X27LE4BHSL4DOUIK - [0:0] - :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-FW-NUKIZ6OKUXPJNT4C - [0:0] :KUBE-MARK-MASQ - [0:0] :KUBE-NODEPORTS - [0:0] :KUBE-POSTROUTING - [0:0] :KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] + :KUBE-SEP-I77PXRDZVX7PMWMN - [0:0] + :KUBE-SEP-OYPFS5VJICHGATKP - [0:0] :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] :KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] :KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-SVC-NUKIZ6OKUXPJNT4C - [0:0] + :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -A PREROUTING -m comment --comment kubernetes service portals -j KUBE-SERVICES @@ -1673,12 +1677,17 @@ func TestTracePackets(t *testing.T) { -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3003 -j KUBE-EXT-X27LE4BHSL4DOUIK + -A KUBE-NODEPORTS -m comment --comment ns5/svc5:p80 -m tcp -p tcp --dport 3002 -j KUBE-EXT-NUKIZ6OKUXPJNT4C -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT - -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 -j KUBE-EXT-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "ns5/svc5:p80 cluster IP" -m tcp -p tcp -d 172.30.0.45 --dport 80 -j KUBE-SVC-NUKIZ6OKUXPJNT4C + -A KUBE-SERVICES -m comment --comment "ns5/svc5:p80 loadbalancer IP" -m tcp -p tcp -d 5.6.7.8 --dport 80 -j KUBE-FW-NUKIZ6OKUXPJNT4C -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-4SW47YFZTEDKD3PK -m comment --comment "masquerade traffic for ns4/svc4:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-4SW47YFZTEDKD3PK -j KUBE-SVC-4SW47YFZTEDKD3PK @@ -1686,14 +1695,18 @@ func TestTracePackets(t *testing.T) { -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT - -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT - -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-EXT-NUKIZ6OKUXPJNT4C -m comment --comment "masquerade traffic for ns5/svc5:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-NUKIZ6OKUXPJNT4C -j KUBE-SVC-NUKIZ6OKUXPJNT4C + -A KUBE-EXT-X27LE4BHSL4DOUIK -m comment --comment "masquerade traffic for ns3/svc3:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-X27LE4BHSL4DOUIK -j KUBE-SVC-X27LE4BHSL4DOUIK + -A KUBE-FW-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-NUKIZ6OKUXPJNT4C + -A KUBE-FW-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 loadbalancer IP" -j KUBE-MARK-DROP -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 + -A KUBE-SEP-I77PXRDZVX7PMWMN -m comment --comment ns5/svc5:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-I77PXRDZVX7PMWMN -m comment --comment ns5/svc5:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3 -j KUBE-MARK-MASQ + -A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ @@ -1705,6 +1718,10 @@ func TestTracePackets(t *testing.T) { -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.5:80" -j KUBE-SEP-C6EBXVWJJZMIWKLZ -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 -> 10.180.0.2:80" -j KUBE-SEP-RS4RBKLTHTF2IUXJ + -A KUBE-SVC-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 cluster IP" -m tcp -p tcp -d 172.30.0.45 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-NUKIZ6OKUXPJNT4C -m comment --comment "ns5/svc5:p80 -> 10.180.0.3:80" -j KUBE-SEP-I77PXRDZVX7PMWMN + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 172.30.0.43 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 -> 10.180.0.3:80" -j KUBE-SEP-OYPFS5VJICHGATKP -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP @@ -1764,7 +1781,7 @@ func TestTracePackets(t *testing.T) { { name: "REJECT", sourceIP: "10.0.0.2", - destIP: "172.30.0.43", + destIP: "172.30.0.46", destPort: 80, output: "REJECT", }, From 95705350d5ba2e7c7e3a5d393818237409382ed0 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 13 Apr 2022 10:04:56 -0400 Subject: [PATCH 3/3] proxy/iptables: Don't use KUBE-MARK-DROP for "no local endpoints" Rather than marking packets to be dropped in the "nat" table and then dropping them from the "filter" table later, just use rules in "filter" to drop the packets we don't like directly. --- pkg/proxy/iptables/proxier.go | 126 ++++++++++++++++++++--------- pkg/proxy/iptables/proxier_test.go | 42 +++++----- 2 files changed, 113 insertions(+), 55 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index ac6a28fdafb..6b759ece813 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1029,33 +1029,79 @@ func (proxier *Proxier) syncProxyRules() { clusterPolicyChain := svcInfo.clusterPolicyChainName localPolicyChain := svcInfo.localPolicyChainName - // These chains designate which policy chain to use for internal- and - // external-destination traffic. + // internalPolicyChain is the chain containing the endpoints for + // "internal" (ClusterIP) traffic. internalTrafficChain is the chain that + // internal traffic is routed to (which is always the same as + // internalPolicyChain). hasInternalEndpoints is true if we should + // generate rules pointing to internalTrafficChain, or false if there are + // no available internal endpoints. internalPolicyChain := clusterPolicyChain - externalPolicyChain := clusterPolicyChain + hasInternalEndpoints := hasEndpoints if svcInfo.InternalPolicyLocal() { internalPolicyChain = localPolicyChain + if len(localEndpoints) == 0 { + hasInternalEndpoints = false + } } + internalTrafficChain := internalPolicyChain + + // Similarly, externalPolicyChain is the chain containing the endpoints + // for "external" (NodePort, LoadBalancer, and ExternalIP) traffic. + // externalTrafficChain is the chain that external traffic is routed to + // (which is always the service's "EXT" chain). hasExternalEndpoints is + // true if there are endpoints that will be reached by external traffic. + // (But we may still have to generate externalTrafficChain even if there + // are no external endpoints, to ensure that the short-circuit rules for + // local traffic are set up.) + externalPolicyChain := clusterPolicyChain + hasExternalEndpoints := hasEndpoints if svcInfo.ExternalPolicyLocal() { externalPolicyChain = localPolicyChain + if len(localEndpoints) == 0 { + hasExternalEndpoints = false + } } - - // These chains are where *ALL* rules which match traffic that is - // service-destined should jump. ClusterIP traffic is considered - // "internal" while NodePort, LoadBalancer, and ExternalIPs traffic is - // considered "external". - internalTrafficChain := internalPolicyChain externalTrafficChain := svcInfo.externalChainName // eventually jumps to externalPolicyChain + var internalTrafficFilterTarget, internalTrafficFilterComment string + var externalTrafficFilterTarget, externalTrafficFilterComment string + if !hasEndpoints { + // The service has no endpoints at all; hasInternalEndpoints and + // hasExternalEndpoints will also be false, and we will not + // generate any chains in the "nat" table for the service; only + // rules in the "filter" table rejecting incoming packets for + // the service's IPs. + internalTrafficFilterTarget = "REJECT" + internalTrafficFilterComment = fmt.Sprintf(`"%s has no endpoints"`, svcPortNameString) + externalTrafficFilterTarget = "REJECT" + externalTrafficFilterComment = internalTrafficFilterComment + } else { + if !hasInternalEndpoints { + // The internalTrafficPolicy is "Local" but there are no local + // endpoints. Traffic to the clusterIP will be dropped, but + // external traffic may still be accepted. + internalTrafficFilterTarget = "DROP" + internalTrafficFilterComment = fmt.Sprintf(`"%s has no local endpoints"`, svcPortNameString) + } + if !hasExternalEndpoints { + // The externalTrafficPolicy is "Local" but there are no + // local endpoints. Traffic to "external" IPs from outside + // the cluster will be dropped, but traffic from inside + // the cluster may still be accepted. + externalTrafficFilterTarget = "DROP" + externalTrafficFilterComment = fmt.Sprintf(`"%s has no local endpoints"`, svcPortNameString) + } + } + // Declare the clusterPolicyChain if needed. - if hasEndpoints && svcInfo.UsesClusterEndpoints() { + if len(clusterEndpoints) > 0 && svcInfo.UsesClusterEndpoints() { // Create the Cluster traffic policy chain proxier.natChains.Write(utiliptables.MakeChainLine(clusterPolicyChain)) activeNATChains[clusterPolicyChain] = true } // Declare the localPolicyChain if needed. - if hasEndpoints && svcInfo.UsesLocalEndpoints() { + if len(localEndpoints) > 0 && svcInfo.UsesLocalEndpoints() { proxier.natChains.Write(utiliptables.MakeChainLine(localPolicyChain)) activeNATChains[localPolicyChain] = true } @@ -1063,7 +1109,9 @@ func (proxier *Proxier) syncProxyRules() { // If any "external" destinations are enabled, set up external traffic // handling. All captured traffic for all external destinations should // jump to externalTrafficChain, which will handle some special-cases - // and then jump to externalPolicyChain. + // and then jump to externalPolicyChain. (We check hasEndpoints here not + // hasExternalEndpoints because we need the short-circuit rules in the + // EXT chain even when there are no external endpoints.) if hasEndpoints && svcInfo.ExternallyAccessible() { proxier.natChains.Write(utiliptables.MakeChainLine(externalTrafficChain)) activeNATChains[externalTrafficChain] = true @@ -1111,13 +1159,15 @@ func (proxier *Proxier) syncProxyRules() { } // Anything else falls thru to the appropriate policy chain. - proxier.natRules.Write( - "-A", string(externalTrafficChain), - "-j", string(externalPolicyChain)) + if hasExternalEndpoints { + proxier.natRules.Write( + "-A", string(externalTrafficChain), + "-j", string(externalPolicyChain)) + } } // Capture the clusterIP. - if hasEndpoints { + if hasInternalEndpoints { args = append(args[:0], "-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcPortNameString), "-m", protocol, "-p", protocol, @@ -1148,11 +1198,11 @@ func (proxier *Proxier) syncProxyRules() { // No endpoints. proxier.filterRules.Write( "-A", string(kubeServicesChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcPortNameString), + "-m", "comment", "--comment", internalTrafficFilterComment, "-m", protocol, "-p", protocol, "-d", svcInfo.ClusterIP().String(), "--dport", strconv.Itoa(svcInfo.Port()), - "-j", "REJECT", + "-j", internalTrafficFilterTarget, ) } @@ -1168,16 +1218,18 @@ func (proxier *Proxier) syncProxyRules() { "-d", externalIP, "--dport", strconv.Itoa(svcInfo.Port()), "-j", string(externalTrafficChain)) - - } else { - // No endpoints. + } + if !hasExternalEndpoints { + // Either no endpoints at all (REJECT) or no endpoints for + // external traffic (DROP anything that didn't get + // short-circuited by the EXT chain.) proxier.filterRules.Write( "-A", string(kubeExternalServicesChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcPortNameString), + "-m", "comment", "--comment", externalTrafficFilterComment, "-m", protocol, "-p", protocol, "-d", externalIP, "--dport", strconv.Itoa(svcInfo.Port()), - "-j", "REJECT", + "-j", externalTrafficFilterTarget, ) } } @@ -1251,16 +1303,19 @@ func (proxier *Proxier) syncProxyRules() { "-j", string(nextChain)) } - } else { - // No endpoints. + } + if !hasExternalEndpoints { + // Either no endpoints at all (REJECT) or no endpoints for + // external traffic (DROP anything that didn't get short-circuited + // by the EXT chain.) for _, lbip := range svcInfo.LoadBalancerIPStrings() { proxier.filterRules.Write( "-A", string(kubeExternalServicesChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcPortNameString), + "-m", "comment", "--comment", externalTrafficFilterComment, "-m", protocol, "-p", protocol, "-d", lbip, "--dport", strconv.Itoa(svcInfo.Port()), - "-j", "REJECT", + "-j", externalTrafficFilterTarget, ) } } @@ -1277,15 +1332,18 @@ func (proxier *Proxier) syncProxyRules() { "-m", protocol, "-p", protocol, "--dport", strconv.Itoa(svcInfo.NodePort()), "-j", string(externalTrafficChain)) - } else { - // No endpoints. + } + if !hasExternalEndpoints { + // Either no endpoints at all (REJECT) or no endpoints for + // external traffic (DROP anything that didn't get + // short-circuited by the EXT chain.) proxier.filterRules.Write( "-A", string(kubeExternalServicesChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcPortNameString), + "-m", "comment", "--comment", externalTrafficFilterComment, "-m", "addrtype", "--dst-type", "LOCAL", "-m", protocol, "-p", protocol, "--dport", strconv.Itoa(svcInfo.NodePort()), - "-j", "REJECT", + "-j", externalTrafficFilterTarget, ) } } @@ -1319,12 +1377,6 @@ func (proxier *Proxier) syncProxyRules() { if svcInfo.ExternalPolicyLocal() { serviceNoLocalEndpointsTotalExternal++ } - // Blackhole all traffic since there are no local endpoints - proxier.natRules.Write( - "-A", string(localPolicyChain), - "-m", "comment", "--comment", - fmt.Sprintf(`"%s has no local endpoints"`, svcPortNameString), - "-j", string(kubeMarkDropChain)) } } } diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 6ea64f49900..3aedb6a4eb3 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1019,6 +1019,9 @@ func TestSortIPTablesRules(t *testing.T) { :KUBE-FORWARD - [0:0] :KUBE-NODEPORTS - [0:0] -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -1091,6 +1094,9 @@ func TestSortIPTablesRules(t *testing.T) { :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -1521,7 +1527,7 @@ func (tracer *iptablesTracer) runChain(table utiliptables.Table, chain utiliptab tracer.markDrop = true continue - case "ACCEPT", "REJECT": + case "ACCEPT", "REJECT", "DROP": // (only valid in filter) tracer.outputs = append(tracer.outputs, rule.Jump.Value) return true @@ -1639,6 +1645,9 @@ func TestTracePackets(t *testing.T) { -A OUTPUT -m conntrack --ctstate NEW -m comment --comment kubernetes service portals -j KUBE-SERVICES -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -A KUBE-SERVICES -m comment --comment "ns6/svc6:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.46 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -1668,7 +1677,6 @@ func TestTracePackets(t *testing.T) { :KUBE-SVC-NUKIZ6OKUXPJNT4C - [0:0] :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -A PREROUTING -m comment --comment kubernetes service portals -j KUBE-SERVICES -A OUTPUT -m comment --comment kubernetes service portals -j KUBE-SERVICES -A POSTROUTING -m comment --comment kubernetes postrouting rules -j KUBE-POSTROUTING @@ -1694,7 +1702,6 @@ func TestTracePackets(t *testing.T) { -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "pod traffic for ns2/svc2:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT - -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT -A KUBE-EXT-NUKIZ6OKUXPJNT4C -m comment --comment "masquerade traffic for ns5/svc5:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-NUKIZ6OKUXPJNT4C -j KUBE-SVC-NUKIZ6OKUXPJNT4C -A KUBE-EXT-X27LE4BHSL4DOUIK -m comment --comment "masquerade traffic for ns3/svc3:p80 external destinations" -j KUBE-MARK-MASQ @@ -1724,7 +1731,6 @@ func TestTracePackets(t *testing.T) { -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 -> 10.180.0.3:80" -j KUBE-SEP-OYPFS5VJICHGATKP -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP COMMIT `) @@ -1765,12 +1771,19 @@ func TestTracePackets(t *testing.T) { masq: true, }, { - name: "KUBE-MARK-DROP", + name: "DROP (via filter table)", sourceIP: testExternalClient, destIP: "192.168.99.22", destPort: 80, output: "DROP", }, + { + name: "DROP (via KUBE-MARK-DROP)", + sourceIP: testExternalClientBlocked, + destIP: "5.6.7.8", + destPort: 80, + output: "DROP", + }, { name: "ACCEPT (NodePortHealthCheck)", sourceIP: testNodeIP, @@ -1956,6 +1969,9 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { :KUBE-FORWARD - [0:0] -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT -A KUBE-SERVICES -m comment --comment "ns6/svc6:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.46 --dport 80 -j REJECT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns2/svc2:p80 has no local endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -1981,7 +1997,6 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { :KUBE-SVC-NUKIZ6OKUXPJNT4C - [0:0] :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3003 -j KUBE-EXT-X27LE4BHSL4DOUIK -A KUBE-NODEPORTS -m comment --comment ns5/svc5:p80 -m tcp -p tcp --dport 3002 -j KUBE-EXT-NUKIZ6OKUXPJNT4C @@ -2000,7 +2015,6 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "pod traffic for ns2/svc2:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT - -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT -A KUBE-EXT-NUKIZ6OKUXPJNT4C -m comment --comment "masquerade traffic for ns5/svc5:p80 external destinations" -j KUBE-MARK-MASQ -A KUBE-EXT-NUKIZ6OKUXPJNT4C -j KUBE-SVC-NUKIZ6OKUXPJNT4C -A KUBE-EXT-X27LE4BHSL4DOUIK -m comment --comment "masquerade traffic for ns3/svc3:p80 external destinations" -j KUBE-MARK-MASQ @@ -2034,7 +2048,6 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 -> 10.180.0.3:80" -j KUBE-SEP-OYPFS5VJICHGATKP -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP COMMIT `) @@ -5226,6 +5239,7 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] + -A KUBE-SERVICES -m comment --comment "ns1/svc1 has no local endpoints" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -5235,15 +5249,11 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { :KUBE-SERVICES - [0:0] :KUBE-MARK-MASQ - [0:0] :KUBE-POSTROUTING - [0:0] - :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] - -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVL-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), flowTests: []packetFlowTest{ @@ -5872,6 +5882,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no local endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -5884,14 +5895,12 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-POSTROUTING - [0:0] :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -5901,7 +5910,6 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), flowTests: []packetFlowTest{ @@ -5959,6 +5967,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] -A KUBE-NODEPORTS -m comment --comment "ns1/svc1 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no local endpoints" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT @@ -5971,14 +5980,12 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { :KUBE-POSTROUTING - [0:0] :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] - :KUBE-SVL-AQI2S6QIMU7PVVRP - [0:0] -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-EXT-AQI2S6QIMU7PVVRP -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "pod traffic for ns1/svc1 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-EXT-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP - -A KUBE-EXT-AQI2S6QIMU7PVVRP -j KUBE-SVL-AQI2S6QIMU7PVVRP -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -5988,7 +5995,6 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.30.1.1 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY - -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), flowTests: []packetFlowTest{