From 36673b618ef4c03de0f442b26fc5dbd802f68054 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 10:20:05 -0500 Subject: [PATCH] Rework masquerade handling in nftables tracer Process the actual rules rather than treating "jump mark-for-masquerade" as having special semantics. --- pkg/proxy/nftables/helpers_test.go | 60 ++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/pkg/proxy/nftables/helpers_test.go b/pkg/proxy/nftables/helpers_test.go index ebf7cd7988a..da0f81b59bb 100644 --- a/pkg/proxy/nftables/helpers_test.go +++ b/pkg/proxy/nftables/helpers_test.go @@ -160,8 +160,9 @@ type nftablesTracer struct { // the return value of tracePacket. outputs []string - // markMasq tracks whether the packet has been marked for masquerading - markMasq bool + // additional info about rules we've hit + markMasq bool + masquerade bool } // newNFTablesTracer creates an nftablesTracer. nodeIPs are the IP to treat as local node @@ -272,7 +273,9 @@ var sourceAddrLocalRegexp = regexp.MustCompile(`^fib saddr type local`) var endpointVMAPRegexp = regexp.MustCompile(`^numgen random mod \d+ vmap \{(.*)\}$`) var endpointVMapEntryRegexp = regexp.MustCompile(`\d+ : goto (\S+)`) -var masqueradeRegexp = regexp.MustCompile(`^jump ` + markMasqChain + `$`) +var masqMarkRegexp = regexp.MustCompile(`^mark set mark or 0x[[:xdigit:]]+$`) +var masqCheckRegexp = regexp.MustCompile(`^mark and 0x[[:xdigit:]]+ != 0 mark set mark xor 0x[[:xdigit:]]+`) +var masqueradeRegexp = regexp.MustCompile(`^masquerade fully-random$`) var jumpRegexp = regexp.MustCompile(`^(jump|goto) (\S+)$`) var returnRegexp = regexp.MustCompile(`^return$`) var verdictRegexp = regexp.MustCompile(`^(drop|reject)$`) @@ -304,10 +307,8 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP for rule != "" { rule = strings.TrimLeft(rule, " ") - // Note that the order of (some of) the cases is important. e.g., - // masqueradeRegexp must be checked before jumpRegexp, since - // jumpRegexp would also match masqueradeRegexp but do the wrong - // thing with it. + // Note that the order of (some of) the cases is important since + // some of the regexes might match parts of others. switch { case destIPOnlyLookupRegexp.MatchString(rule): @@ -424,17 +425,35 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP break } - case masqueradeRegexp.MatchString(rule): - // `^jump mark-for-masquerade$` - // Mark for masquerade: we just treat the jump rule itself as - // being what creates the mark, rather than trying to handle - // the rules inside that chain and the "masquerading" chain. - match := jumpRegexp.FindStringSubmatch(rule) + case masqMarkRegexp.MatchString(rule): + // `^mark set mark or 0x[[:xdigit:]]+$` + // Mark for masquerade. + match := masqMarkRegexp.FindStringSubmatch(rule) rule = strings.TrimPrefix(rule, match[0]) tracer.matches = append(tracer.matches, ruleObj.Rule) tracer.markMasq = true + case masqCheckRegexp.MatchString(rule): + // `^mark and 0x[[:xdigit:]]+ != 0 mark set mark xor 0x[[:xdigit:]]+ + // Checks and clears the mark + match := masqCheckRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if !tracer.markMasq { + rule = "" + break + } + tracer.markMasq = false + + case masqueradeRegexp.MatchString(rule): + // ^masquerade fully-random$ + // Actually perform the masquerading + match := masqueradeRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + + tracer.matches = append(tracer.matches, ruleObj.Rule) + tracer.masquerade = true + case jumpRegexp.MatchString(rule): // `^(jump|goto) (\S+)$` // Jumps to another chain. @@ -545,8 +564,11 @@ func tracePacket(t *testing.T, nft *knftables.Fake, sourceIP, protocol, destIP, tracer.runChain("filter-input", sourceIP, protocol, destIP, destPort) // Skip filter-output-pre-dnat, filter-output and nat-output as they ought to be fully redundant with the prerouting chains. - // Skip nat-postrouting because it only does masquerading and we handle that separately. - return tracer.matches, strings.Join(tracer.outputs, ", "), tracer.markMasq + + // Run nat-postrouting to handle masquerading + tracer.runChain("nat-postrouting", sourceIP, protocol, destIP, destPort) + + return tracer.matches, strings.Join(tracer.outputs, ", "), tracer.masquerade } type packetFlowTest struct { @@ -590,7 +612,7 @@ var testInput = dedent.Dedent(` add chain ip testing mark-for-masquerade add rule ip testing mark-for-masquerade mark set mark or 0x4000 add chain ip testing masquerading - add rule ip testing masquerading mark and 0x4000 != 0 masquerade fully-random + add rule ip testing masquerading mark and 0x4000 != 0 mark set mark xor 0x4000 masquerade fully-random add set ip testing firewall { type ipv4_addr . inet_proto . inet_service ; comment "destinations that are subject to LoadBalancerSourceRanges" ; } add set ip testing firewall-allow { type ipv4_addr . inet_proto . inet_service . ipv4_addr ; flags interval ; comment "destinations+sources that are allowed by LoadBalancerSourceRanges" ; } @@ -655,7 +677,7 @@ var testExpected = dedent.Dedent(` add rule ip testing firewall-allow-check drop add rule ip testing firewall-check ip daddr . meta l4proto . th dport @firewall jump firewall-allow-check add rule ip testing mark-for-masquerade mark set mark or 0x4000 - add rule ip testing masquerading mark and 0x4000 != 0 masquerade fully-random + add rule ip testing masquerading mark and 0x4000 != 0 mark set mark xor 0x4000 masquerade fully-random add rule ip testing service-42NFTM6N-ns2/svc2/tcp/p80 ip daddr 172.30.0.42 tcp dport 80 ip saddr != 10.0.0.0/8 jump mark-for-masquerade add rule ip testing service-42NFTM6N-ns2/svc2/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 } add rule ip testing service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 tcp dport 80 ip saddr != 10.0.0.0/8 jump mark-for-masquerade @@ -801,7 +823,7 @@ func TestTracePacketV4(t *testing.T) { add chain ip kube-proxy firewall-HVFWP5L3-ns5/svc5/tcp/p80 add rule ip kube-proxy mark-for-masquerade mark set mark or 0x4000 - add rule ip kube-proxy masquerading mark and 0x4000 != 0 masquerade fully-random + add rule ip kube-proxy masquerading mark and 0x4000 != 0 mark set mark xor 0x4000 masquerade fully-random add rule ip kube-proxy filter-prerouting-pre-dnat ct state new jump firewall-check add rule ip kube-proxy filter-forward ct state new jump endpoints-check add rule ip kube-proxy filter-input ct state new jump endpoints-check @@ -1022,7 +1044,7 @@ func TestTracePacketV6(t *testing.T) { add rule ip6 kube-proxy filter-prerouting-pre-dnat ct state new jump firewall-check add rule ip6 kube-proxy firewall-check ip6 daddr . meta l4proto . th dport vmap @firewall-ips add rule ip6 kube-proxy mark-for-masquerade mark set mark or 0x4000 - add rule ip6 kube-proxy masquerading mark and 0x4000 != 0 masquerade fully-random + add rule ip6 kube-proxy masquerading mark and 0x4000 != 0 mark set mark xor 0x4000 masquerade fully-random add rule ip6 kube-proxy nat-output jump services add rule ip6 kube-proxy nat-postrouting jump masquerading add rule ip6 kube-proxy nat-prerouting jump services