diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index e3e25d89728..7ccbbff37c1 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -377,6 +377,8 @@ func TestDeleteEndpointConnectionsIPv6(t *testing.T) { const testHostname = "test-hostname" const testNodeIP = "192.168.0.2" +const testExternalClient = "203.0.113.2" +const testExternalClientBlocked = "203.0.113.130" func NewFakeProxier(ipt utiliptables.Interface) *Proxier { // TODO: Call NewProxier after refactoring out the goroutine @@ -1373,6 +1375,591 @@ func assertIPTablesRulesNotEqual(t *testing.T, line int, expected, result string } } +// ruleMatchesIP helps test whether an iptables rule such as "! -s 192.168.0.0/16" matches +// ipStr. ruleAddress is either an IP address ("1.2.3.4") or a CIDR string +// ("1.2.3.0/24"). negated is whether the iptables rule negates the match. +func ruleMatchesIP(t *testing.T, negated bool, ruleAddress, ipStr string) bool { + ip := netutils.ParseIPSloppy(ipStr) + if ip == nil { + t.Fatalf("Bad IP in test case: %s", ipStr) + } + + var matches bool + if strings.Contains(ruleAddress, "/") { + _, cidr, err := netutils.ParseCIDRSloppy(ruleAddress) + if err != nil { + t.Errorf("Bad CIDR in kube-proxy output: %v", err) + } + matches = cidr.Contains(ip) + } else { + ip2 := netutils.ParseIPSloppy(ruleAddress) + if ip2 == nil { + t.Errorf("Bad IP/CIDR in kube-proxy output: %s", ruleAddress) + } + matches = ip.Equal(ip2) + } + return (!negated && matches) || (negated && !matches) +} + +// Regular expressions used by iptablesTracer. Note that these are not fully general-purpose +// and may need to be updated if we make large changes to our iptable rules. +var addRuleToChainRegex = regexp.MustCompile(`-A ([^ ]*) `) +var moduleRegex = regexp.MustCompile("-m ([^ ]*)") +var commentRegex = regexp.MustCompile(`-m comment --comment ("[^"]*"|[^" ]*) `) +var srcLocalRegex = regexp.MustCompile("(!)? --src-type LOCAL") +var destLocalRegex = regexp.MustCompile("(!)? --dst-type LOCAL") +var destIPRegex = regexp.MustCompile("(!)? -d ([^ ]*) ") +var destPortRegex = regexp.MustCompile(" --dport ([^ ]*) ") +var sourceIPRegex = regexp.MustCompile("(!)? -s ([^ ]*) ") +var affinityRegex = regexp.MustCompile(" --rcheck ") + +// (If `--probability` appears, it can only appear before the `-j`, and if `--to-destination` +// appears it can only appear after the `-j`, so this is not as fragile as it looks. +var jumpRegex = regexp.MustCompile("(--probability.*)? -j ([^ ]*)( --to-destination (.*))?$") + +func Test_iptablesTracerRegexps(t *testing.T) { + testCases := []struct { + name string + regex *regexp.Regexp + rule string + matches []string + }{ + { + name: "addRuleToChainRegex", + regex: addRuleToChainRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{`-A KUBE-NODEPORTS `, "KUBE-NODEPORTS"}, + }, + { + name: "addRuleToChainRegex requires an actual rule, not just a chain name", + regex: addRuleToChainRegex, + rule: `-A KUBE-NODEPORTS`, + matches: nil, + }, + { + name: "addRuleToChainRegex only matches adds", + regex: addRuleToChainRegex, + rule: `-D KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: nil, + }, + { + name: "commentRegex with quoted comment", + regex: commentRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{`-m comment --comment "ns2/svc2:p80 health check node port" `, `"ns2/svc2:p80 health check node port"`}, + }, + { + name: "commentRegex with unquoted comment", + regex: commentRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + matches: []string{`-m comment --comment ns1/svc1:p80 `, "ns1/svc1:p80"}, + }, + { + name: "no comment", + regex: commentRegex, + rule: `-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000`, + matches: nil, + }, + { + name: "moduleRegex", + regex: moduleRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{"-m comment", "comment"}, + }, + { + name: "local source", + regex: srcLocalRegex, + rule: `-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ`, + matches: []string{" --src-type LOCAL", ""}, + }, + { + name: "not local destination", + regex: destLocalRegex, + rule: `-A RULE-TYPE-NOT-CURRENTLY-USED-BY-KUBE-PROXY -m addrtype ! --dst-type LOCAL -j KUBE-MARK-MASQ`, + matches: []string{"! --dst-type LOCAL", "!"}, + }, + { + name: "destination IP", + regex: destIPRegex, + rule: `-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`, + matches: []string{" -d 172.30.0.41 ", "", "172.30.0.41"}, + }, + { + name: "destination port", + regex: destPortRegex, + rule: `-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`, + matches: []string{" --dport 80 ", "80"}, + }, + { + name: "destination IP but no port", + regex: destPortRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -d 172.30.0.41 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ`, + matches: nil, + }, + { + name: "source IP", + regex: sourceIPRegex, + rule: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ`, + matches: []string{" -s 10.180.0.1 ", "", "10.180.0.1"}, + }, + { + name: "not source IP", + regex: sourceIPRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ`, + matches: []string{"! -s 10.0.0.0/8 ", "!", "10.0.0.0/8"}, + }, + { + name: "affinityRegex", + regex: affinityRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + matches: []string{" --rcheck "}, + }, + { + name: "jump to internal target", + regex: jumpRegex, + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + matches: []string{" -j ACCEPT", "", "ACCEPT", "", ""}, + }, + { + name: "jump to KUBE chain", + regex: jumpRegex, + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + matches: []string{" -j KUBE-SEP-SXIVWICOYRO3J4NJ", "", "KUBE-SEP-SXIVWICOYRO3J4NJ", "", ""}, + }, + { + name: "jump to DNAT", + regex: jumpRegex, + rule: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80`, + matches: []string{" -j DNAT --to-destination 10.180.0.1:80", "", "DNAT", " --to-destination 10.180.0.1:80", "10.180.0.1:80"}, + }, + { + name: "jump to endpoint", + regex: jumpRegex, + rule: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + matches: []string{"--probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC", "--probability 0.5000000000", "KUBE-SEP-UKSFD7AGPMPPLUHC", "", ""}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + matches := testCase.regex.FindStringSubmatch(testCase.rule) + if !reflect.DeepEqual(matches, testCase.matches) { + t.Errorf("bad match: expected %#v, got %#v", testCase.matches, matches) + } + }) + } +} + +// knownModules is the set of modules (ie "-m foo") that we allow to be present in rules passed +// to an iptablesTracer. If a rule using another module is found in a rule, the test will +// fail. +// +// If a module is in knownModules but is not in noMatchModules and is not handled by +// ruleMatches, then the result is that match rules using that module will have no effect +// for tracing purposes. +var knownModules = sets.NewString("addrtype", "comment", "conntrack", "mark", "recent", "statistic", "tcp", "udp") + +// noMatchModules is the list of modules that if we see them in a rule, we just +// assume the rule doesn't match and ignore it (because rules with these modules exist +// in the data we are testing against, but aren't relevant to what we're testing). +var noMatchModules = sets.NewString("conntrack", "mark") + +type iptablesChain []string +type iptablesTable map[string]iptablesChain + +// iptablesTracer holds data used while virtually tracing a packet through a set of +// iptables rules +type iptablesTracer struct { + tables map[string]iptablesTable + nodeIP string + t *testing.T + + // matches accumulates the list of rules that were matched, for debugging purposes. + matches []string + + // outputs accumulates the list of matched terminal rule targets (endpoint + // IP:ports, or a special target like "REJECT") and is eventually used to generate + // the return value of tracePacket. + outputs []string + + // markMasq and markDrop track whether the packet has been marked for masquerading + // or dropping. + markMasq bool + markDrop bool +} + +// newIPTablesTracer creates an iptablesTracer. ruleData is an iptables rule dump (as with +// "iptables-save"). nodeIP is the IP to treat as the local node IP (for determining +// whether rules with "--src-type LOCAL" or "--dst-type LOCAL" match). +func newIPTablesTracer(t *testing.T, ruleData, nodeIP string) (*iptablesTracer, error) { + tables, err := parseIPTablesData(ruleData) + if err != nil { + return nil, err + } + + tracer := &iptablesTracer{ + tables: make(map[string]iptablesTable), + nodeIP: nodeIP, + t: t, + } + + for name, rules := range tables { + table := make(iptablesTable) + for _, rule := range rules { + match := addRuleToChainRegex.FindStringSubmatch(rule) + if match != nil { + chainName := match[1] + table[chainName] = append(table[chainName], rule) + } + } + tracer.tables[name] = table + } + + return tracer, nil +} + +// ruleMatches checks if the given iptables rule matches (at least probabilistically) a +// packet with the given sourceIP, destIP, and destPort. (Note that protocol is currently +// ignored.) +func (tracer *iptablesTracer) ruleMatches(rule, sourceIP, destIP, destPort string) bool { + var match []string + + // Delete comments so we don't mistakenly match something in a comment string later + rule = commentRegex.ReplaceAllString(rule, "") + + // Make sure the rule only uses modules ("-m foo") that we are aware of + for _, matches := range moduleRegex.FindAllStringSubmatch(rule, -1) { + moduleName := matches[1] + if !knownModules.Has(moduleName) { + tracer.t.Errorf("Rule %q uses unknown iptables module %q", rule, moduleName) + } + if noMatchModules.Has(moduleName) { + // This rule is doing something irrelevant to iptablesTracer + return false + } + } + + // The sub-rules within an iptables rule are ANDed together, so the rule only + // matches if all of them match. So go through the subrules, and if any of them + // DON'T match, then fail. + + // Match local/non-local. + match = srcLocalRegex.FindStringSubmatch(rule) + if match != nil { + wantLocal := (match[1] != "!") + sourceIsLocal := (sourceIP == tracer.nodeIP || sourceIP == "127.0.0.1") + if wantLocal != sourceIsLocal { + return false + } + } + match = destLocalRegex.FindStringSubmatch(rule) + if match != nil { + wantLocal := (match[1] != "!") + destIsLocal := (destIP == tracer.nodeIP || destIP == "127.0.0.1") + if wantLocal != destIsLocal { + return false + } + } + + // Match destination IP/port. + match = destIPRegex.FindStringSubmatch(rule) + if match != nil { + negated := match[1] == "!" + ruleAddress := match[2] + if !ruleMatchesIP(tracer.t, negated, ruleAddress, destIP) { + return false + } + } + match = destPortRegex.FindStringSubmatch(rule) + if match != nil { + rulePort := match[1] + if rulePort != destPort { + return false + } + } + + // Match source IP (but not currently port) + match = sourceIPRegex.FindStringSubmatch(rule) + if match != nil { + negated := match[1] == "!" + ruleAddress := match[2] + if !ruleMatchesIP(tracer.t, negated, ruleAddress, sourceIP) { + return false + } + } + + // The iptablesTracer has no state/history, so any rule that checks whether affinity + // has been established for a particular endpoint must not match. + if affinityRegex.MatchString(rule) { + return false + } + + // Anything else is assumed to match + return true +} + +// runChain runs the given packet through the rules in the given table and chain, updating +// tracer's internal state accordingly. It returns true if it hits a terminal action. +func (tracer *iptablesTracer) runChain(table, chain, sourceIP, destIP, destPort string) bool { + for _, rule := range tracer.tables[table][chain] { + match := jumpRegex.FindStringSubmatch(rule) + if match == nil { + // You _can_ have rules that don't end in `-j`, but we don't currently + // do that. + tracer.t.Errorf("Could not find jump target in rule %q", rule) + } + isProbabilisticMatch := (match[1] != "") + target := match[2] + natDestination := match[4] + + if !tracer.ruleMatches(rule, sourceIP, destIP, destPort) { + continue + } + // record the matched rule for debugging purposes + tracer.matches = append(tracer.matches, rule) + + switch target { + case "KUBE-MARK-MASQ": + tracer.markMasq = true + continue + + case "KUBE-MARK-DROP": + tracer.markDrop = true + continue + + case "ACCEPT", "REJECT": + // (only valid in filter) + tracer.outputs = append(tracer.outputs, target) + return true + + case "DNAT": + // (only valid in nat) + tracer.outputs = append(tracer.outputs, natDestination) + return true + + default: + // We got a "-j KUBE-SOMETHING", so process that chain + terminated := tracer.runChain(table, target, sourceIP, destIP, destPort) + + // If the subchain hit a terminal rule AND the rule that sent us + // to that chain was non-probabilistic, then this chain terminates + // as well. But if we went there because of a --probability rule, + // then we want to keep accumulating further matches against this + // chain. + if terminated && !isProbabilisticMatch { + return true + } + } + } + + return false +} + +// tracePacket determines what would happen to a packet with the given sourceIP, destIP, +// and destPort, given the indicated iptables ruleData. nodeIP is the local node IP (for +// rules matching "LOCAL"). +// +// The return values are: an array of matched rules (for debugging), the final packet +// destinations (a comma-separated list of IPs, or one of the special targets "ACCEPT", +// "DROP", or "REJECT"), and whether the packet would be masqueraded. +func tracePacket(t *testing.T, ruleData, sourceIP, destIP, destPort, nodeIP string) ([]string, string, bool) { + tracer, err := newIPTablesTracer(t, ruleData, nodeIP) + if err != nil { + t.Errorf("Bad iptables ruleData: %v", err) + } + + // nat:PREROUTING goes first, then the filter chains, then nat:POSTROUTING. For our + // purposes that means we run through the "nat" chains first, starting from the top of + // KUBE-SERVICES, then we do the "filter" chains. The only interesting thing that + // happens in nat:POSTROUTING is that the masquerade mark gets turned into actual + // masquerading. + + // FIXME: we ought to be able to say + // trace.runChain("nat", "PREROUTING", ...) + // here instead of + // trace.runChain("nat", "KUBE-SERVICES", ...) + // (and similarly below with the "filter" chains) but this doesn't work because the + // rules like "-A PREROUTING -j KUBE-SERVICES" are created with iptables.EnsureRule(), + // which iptablestest.FakeIPTables doesn't implement, so those rules will be missing + // from the ruleData we have. So we have to explicitly specify each kube-proxy chain + // we want to run through here. + tracer.runChain("nat", "KUBE-SERVICES", sourceIP, destIP, destPort) + + // Process pending DNAT (which theoretically might affect REJECT/ACCEPT filter rules) + if len(tracer.outputs) != 0 { + destIP = strings.Split(tracer.outputs[0], ":")[0] + } + + // Now run the filter rules to see if the packet is REJECTed or ACCEPTed. The DROP + // rule is created by kubelet, not us, so we have to simulate that manually + if tracer.markDrop { + return tracer.matches, "DROP", false + } + tracer.runChain("filter", "KUBE-SERVICES", sourceIP, destIP, destPort) + tracer.runChain("filter", "KUBE-EXTERNAL-SERVICES", sourceIP, destIP, destPort) + tracer.runChain("filter", "KUBE-NODEPORTS", sourceIP, destIP, destPort) + + return tracer.matches, strings.Join(tracer.outputs, ", "), tracer.markMasq +} + +type packetFlowTest struct { + name string + sourceIP string + destIP string + destPort int + output string + masq bool +} + +func runPacketFlowTests(t *testing.T, line int, ruleData, nodeIP string, testCases []packetFlowTest) { + lineStr := "" + if line != 0 { + lineStr = fmt.Sprintf(" (from line %d)", line) + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + matches, output, masq := tracePacket(t, ruleData, tc.sourceIP, tc.destIP, fmt.Sprintf("%d", tc.destPort), nodeIP) + var errors []string + if output != tc.output { + errors = append(errors, fmt.Sprintf("wrong output: expected %q got %q", tc.output, output)) + } + if masq != tc.masq { + errors = append(errors, fmt.Sprintf("wrong masq: expected %v got %v", tc.masq, masq)) + } + if errors != nil { + t.Errorf("Test %q of a packet from %s to %s:%d%s got result:\n%s\n\nBy matching:\n%s\n\n", + tc.name, tc.sourceIP, tc.destIP, tc.destPort, lineStr, strings.Join(errors, "\n"), strings.Join(matches, "\n")) + } + }) + } +} + +// This tests tracePackets against static data, just to make sure we match things in the +// way we expect to. +func TestTracePackets(t *testing.T) { + rules := dedent.Dedent(` + *filter + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-SERVICES - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 has no endpoints" -m tcp -p tcp -d 172.30.0.43 --dport 80 -j REJECT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-EXT-4SW47YFZTEDKD3PK - [0:0] + :KUBE-EXT-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-EXT-PAZTZYUUMV5KCDZL - [0:0] + :KUBE-EXT-X27LE4BHSL4DOUIK - [0:0] + :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] + :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + :KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] + :KUBE-SERVICES - [0:0] + :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] + :KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SVL-GNZBNJ2PO5MGZ6GT - [0:0] + -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 192.168.99.22 --dport 80 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 192.168.99.33 --dport 80 -j KUBE-EXT-4SW47YFZTEDKD3PK + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + -A KUBE-EXT-4SW47YFZTEDKD3PK -m comment --comment "masquerade traffic for ns4/svc4:p80 external destinations" -j KUBE-MARK-MASQ + -A KUBE-EXT-4SW47YFZTEDKD3PK -j KUBE-SVC-4SW47YFZTEDKD3PK + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "pod traffic for ns2/svc2:p80 external destinations" -s 10.0.0.0/8 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 external destinations" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT + -A KUBE-EXT-GNZBNJ2PO5MGZ6GT -j KUBE-SVL-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 203.0.113.0/25 -j KUBE-EXT-GNZBNJ2PO5MGZ6GT + -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5 -j KUBE-MARK-MASQ + -A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2 -j KUBE-MARK-MASQ + -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4 -j KUBE-MARK-MASQ + -A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 172.30.0.44 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.4:80" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC + -A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 -> 10.180.0.5:80" -j KUBE-SEP-C6EBXVWJJZMIWKLZ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 172.30.0.42 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 -> 10.180.0.2:80" -j KUBE-SEP-RS4RBKLTHTF2IUXJ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 -> 10.180.0.1:80" -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SVL-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP + COMMIT + `) + + runPacketFlowTests(t, getLine(), rules, testNodeIP, []packetFlowTest{ + { + name: "no match", + sourceIP: "10.0.0.2", + destIP: "10.0.0.3", + destPort: 80, + output: "", + }, + { + name: "single endpoint", + sourceIP: "10.0.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + }, + { + name: "multiple endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.0.44", + destPort: 80, + output: "10.180.0.4:80, 10.180.0.5:80", + }, + { + name: "LOCAL, KUBE-MARK-MASQ", + sourceIP: testNodeIP, + destIP: "192.168.99.22", + destPort: 80, + output: "10.180.0.2:80", + masq: true, + }, + { + name: "KUBE-MARK-DROP", + sourceIP: testExternalClient, + destIP: "192.168.99.22", + destPort: 80, + output: "DROP", + }, + { + name: "ACCEPT (NodePortHealthCheck)", + sourceIP: testNodeIP, + destIP: testNodeIP, + destPort: 30000, + output: "ACCEPT", + }, + { + name: "REJECT", + sourceIP: "10.0.0.2", + destIP: "172.30.0.43", + destPort: 80, + output: "REJECT", + }, + }) +} + // TestOverallIPTablesRulesWithMultipleServices creates 4 types of services: ClusterIP, // LoadBalancer, ExternalIP and NodePort and verifies if the NAT table rules created // are exactly the same as what is expected. This test provides an overall view of how @@ -1673,6 +2260,16 @@ func TestClusterIPReject(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP rejected", + sourceIP: "10.0.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "REJECT", + }, + }) } func TestClusterIPEndpointsJump(t *testing.T) { @@ -1745,6 +2342,25 @@ func TestClusterIPEndpointsJump(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP accepted", + sourceIP: "10.180.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: false, + }, + { + name: "hairpin to cluster IP", + sourceIP: "10.180.0.1", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + }) } func TestLoadBalancer(t *testing.T) { @@ -1836,6 +2452,62 @@ func TestLoadBalancer(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: false, + }, + { + name: "external to nodePort", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "nodePort bypasses LoadBalancerSourceRanges", + sourceIP: testExternalClientBlocked, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "accepted external to LB", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "blocked external to LB", + sourceIP: testExternalClientBlocked, + destIP: svcLBIP, + destPort: svcPort, + output: "DROP", + }, + { + name: "pod to LB (blocked by LoadBalancerSourceRanges)", + sourceIP: "10.0.0.2", + destIP: svcLBIP, + destPort: svcPort, + output: "DROP", + }, + { + name: "node to LB (blocked by LoadBalancerSourceRanges)", + sourceIP: testNodeIP, + destIP: svcLBIP, + destPort: svcPort, + output: "DROP", + }, + }) } func TestNodePort(t *testing.T) { @@ -1915,6 +2587,41 @@ func TestNodePort(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: false, + }, + { + name: "external to nodePort", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "node to nodePort", + sourceIP: testNodeIP, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + { + name: "localhost to nodePort gets masqueraded", + sourceIP: "127.0.0.1", + destIP: "127.0.0.1", + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP, svcPort), + masq: true, + }, + }) } func TestHealthCheckNodePort(t *testing.T) { @@ -1932,21 +2639,19 @@ func TestHealthCheckNodePort(t *testing.T) { Protocol: v1.ProtocolTCP, } - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "LoadBalancer" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - svc.Spec.HealthCheckNodePort = int32(svcHealthCheckNodePort) - svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal - }), - ) - + svc := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + svc.Spec.HealthCheckNodePort = int32(svcHealthCheckNodePort) + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + }) + makeServiceMap(fp, svc) fp.syncProxyRules() expected := dedent.Dedent(` @@ -1976,6 +2681,30 @@ func TestHealthCheckNodePort(t *testing.T) { `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "firewall accepts HealthCheckNodePort", + sourceIP: "1.2.3.4", + destIP: testNodeIP, + destPort: svcHealthCheckNodePort, + output: "ACCEPT", + masq: false, + }, + }) + + fp.OnServiceDelete(svc) + fp.syncProxyRules() + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "HealthCheckNodePort no longer has any rule", + sourceIP: "1.2.3.4", + destIP: testNodeIP, + destPort: svcHealthCheckNodePort, + output: "", + }, + }) } func TestMasqueradeRule(t *testing.T) { @@ -2068,6 +2797,23 @@ func TestExternalIPsReject(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP with no endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "external IP with no endpoints", + sourceIP: testExternalClient, + destIP: svcExternalIPs, + destPort: svcPort, + output: "REJECT", + }, + }) } func TestOnlyLocalExternalIPs(t *testing.T) { @@ -2159,6 +2905,25 @@ func TestOnlyLocalExternalIPs(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "cluster IP hits both endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external IP hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: svcExternalIPs, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + }) } // TestNonLocalExternalIPs tests if we add the masquerade rule into svcChain in order to @@ -2247,6 +3012,25 @@ func TestNonLocalExternalIPs(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to external IP", + sourceIP: testExternalClient, + destIP: svcExternalIPs, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: true, + }, + }) } func TestNodePortReject(t *testing.T) { @@ -2300,6 +3084,30 @@ func TestNodePortReject(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "pod to NodePort", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: "REJECT", + }, + { + name: "external to NodePort", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: "REJECT", + }, + }) } func TestLoadBalancerReject(t *testing.T) { @@ -2367,6 +3175,30 @@ func TestLoadBalancerReject(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "pod to LoadBalancer IP", + sourceIP: "10.0.0.2", + destIP: svcLBIP, + destPort: svcPort, + output: "REJECT", + }, + { + name: "external to LoadBalancer IP", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: "REJECT", + }, + }) } func TestOnlyLocalLoadBalancing(t *testing.T) { @@ -2475,6 +3307,33 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP hits both endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to LB IP hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: svcLBIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + { + name: "external to NodePort hits only local endpoint, unmasqueraded", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + }) } func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { @@ -2622,6 +3481,59 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable fp.syncProxyRules() assertIPTablesRulesEqual(t, line, expected, fp.iptablesData.String()) + + runPacketFlowTests(t, line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP hit both endpoints", + sourceIP: "10.0.0.2", + destIP: svcIP, + destPort: svcPort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + { + name: "external to NodePort hits only local endpoint", + sourceIP: testExternalClient, + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + { + name: "pod to localhost doesn't work because localhost is not in nodePortAddresses", + sourceIP: "10.0.0.2", + destIP: "127.0.0.1", + destPort: svcNodePort, + output: "", + }, + }) + + if fp.localDetector.IsImplemented() { + // pod-to-NodePort is treated as internal traffic, so we see both endpoints + runPacketFlowTests(t, line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to NodePort hits both endpoints", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d, %s:%d", epIP1, svcPort, epIP2, svcPort), + masq: false, + }, + }) + } else { + // pod-to-NodePort is (incorrectly) treated as external traffic + // when there is no LocalTrafficDetector. + runPacketFlowTests(t, line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to NodePort hits only local endpoint", + sourceIP: "10.0.0.2", + destIP: testNodeIP, + destPort: svcNodePort, + output: fmt.Sprintf("%s:%d", epIP2, svcPort), + masq: false, + }, + }) + } } func TestComputeProbability(t *testing.T) { @@ -4303,6 +5215,7 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { endpoints []endpoint expectEndpointRule bool expectedIPTablesWithSlice string + flowTests []packetFlowTest }{ { name: "internalTrafficPolicy is cluster", @@ -4316,6 +5229,16 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { }, expectEndpointRule: true, expectedIPTablesWithSlice: clusterExpectedIPTables, + flowTests: []packetFlowTest{ + { + name: "pod to ClusterIP hits all endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.3:80", + masq: false, + }, + }, }, { name: "internalTrafficPolicy is local and there are local endpoints", @@ -4357,6 +5280,16 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.1:80" -j KUBE-SEP-3JOIVZTXZZRGORX4 COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to ClusterIP hits only local endpoint", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80", + masq: false, + }, + }, }, { name: "internalTrafficPolicy is local and there are no local endpoints", @@ -4395,6 +5328,15 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), + flowTests: []packetFlowTest{ + { + name: "no endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "DROP", + }, + }, }, { name: "Local internalTrafficPolicy is ignored when feature gate is off", @@ -4408,6 +5350,16 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { }, expectEndpointRule: false, expectedIPTablesWithSlice: clusterExpectedIPTables, + flowTests: []packetFlowTest{ + { + name: "pod to ClusterIP hits all endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.3:80", + masq: false, + }, + }, }, } @@ -4461,12 +5413,24 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() assertIPTablesRulesEqual(t, tc.line, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) + runPacketFlowTests(t, tc.line, fp.iptablesData.String(), testNodeIP, tc.flowTests) + fp.OnEndpointSliceDelete(endpointSlice) + fp.syncProxyRules() if tc.expectEndpointRule { fp.OnEndpointSliceDelete(endpointSlice) fp.syncProxyRules() assertIPTablesRulesNotEqual(t, tc.line, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) } + runPacketFlowTests(t, tc.line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "endpoints deleted", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + }) }) } } @@ -4515,6 +5479,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { endpointslice *discovery.EndpointSlice expectedIPTables string noUsableEndpoints bool + flowTests []packetFlowTest }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", @@ -4635,6 +5600,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -j KUBE-SEP-IO5XOSKPAXIFQXAJ COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80", + masq: false, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist", @@ -4755,6 +5738,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.2:80" -j KUBE-SEP-IO5XOSKPAXIFQXAJ COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80", + masq: false, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist", @@ -4863,6 +5864,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.3:80" -j KUBE-SEP-XGJFVO3L2O5SRFNT COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.2:80, 10.0.1.3:80", + masq: false, + }, + }, }, { name: "with ProxyTerminatingEndpoints disabled, only non-local and terminating endpoints exist", @@ -4967,6 +5986,23 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "DROP", + }, + }, }, { name: "ProxyTerminatingEndpoints enabled, terminating endpoints on remote node", @@ -5037,6 +6073,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-SVL-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + }, + { + name: "external to LB, no locally-usable endpoints", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "DROP", + }, + }, }, { name: "no usable endpoints on any node", @@ -5103,6 +6155,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP, no usable endpoints", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB, no usable endpoints", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }, }, } @@ -5120,6 +6188,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() @@ -5129,6 +6198,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyLocal(t *testing.T) { } else { assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to clusterIP after endpoints deleted", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB after endpoints deleted", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }) }) } } @@ -5177,6 +6262,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) endpointslice *discovery.EndpointSlice expectedIPTables string noUsableEndpoints bool + flowTests []packetFlowTest }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", @@ -5288,6 +6374,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: true, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist", @@ -5399,6 +6503,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.1:80, 10.0.1.2:80, 10.0.1.5:80", + masq: true, + }, + }, }, { name: "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist", @@ -5503,6 +6625,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.2:80, 10.0.1.3:80, 10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.2:80, 10.0.1.3:80, 10.0.1.5:80", + masq: true, + }, + }, }, { name: "with ProxyTerminatingEndpoints disabled, only terminating endpoints exist", @@ -5593,6 +6733,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }, }, { name: "ProxyTerminatingEndpoints enabled, terminating endpoints on remote node", @@ -5656,6 +6812,24 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 -> 10.0.1.5:80" -j KUBE-SEP-EQCHZ7S2PJ72OHAY COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "10.0.1.5:80", + masq: false, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.0.1.5:80", + masq: true, + }, + }, }, { name: "no usable endpoints on any node", @@ -5721,6 +6895,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE COMMIT `), + flowTests: []packetFlowTest{ + { + name: "pod to clusterIP", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }, }, } @@ -5738,6 +6928,7 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() assertIPTablesRulesEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, testcase.flowTests) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() @@ -5747,6 +6938,22 @@ func TestEndpointSliceWithTerminatingEndpointsTrafficPolicyCluster(t *testing.T) } else { assertIPTablesRulesNotEqual(t, testcase.line, testcase.expectedIPTables, fp.iptablesData.String()) } + runPacketFlowTests(t, testcase.line, fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to clusterIP after endpoints deleted", + sourceIP: "10.0.0.2", + destIP: "172.30.1.1", + destPort: 80, + output: "REJECT", + }, + { + name: "external to LB after endpoints deleted", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "REJECT", + }, + }) }) } } @@ -5824,6 +7031,41 @@ func TestMasqueradeAll(t *testing.T) { COMMIT `) assertIPTablesRulesEqual(t, getLine(), expected, fp.iptablesData.String()) + + runPacketFlowTests(t, getLine(), fp.iptablesData.String(), testNodeIP, []packetFlowTest{ + { + name: "pod to cluster IP", + sourceIP: "10.0.0.2", + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + { + name: "pod to LB", + sourceIP: "10.0.0.2", + destIP: "1.2.3.4", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + { + name: "external to cluster IP", + sourceIP: testExternalClient, + destIP: "172.30.0.41", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + { + name: "external to LB", + sourceIP: testExternalClient, + destIP: "1.2.3.4", + destPort: 80, + output: "10.180.0.1:80", + masq: true, + }, + }) } func countEndpointsAndComments(iptablesData string, matchEndpoint string) (string, int, int) {