From 08680192fbff41f59ab70d05d471db5055a9fed9 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 4 Nov 2021 16:30:12 -0400 Subject: [PATCH 1/8] proxy/iptables: Fix sync_proxy_rules_iptables_total metric It was counting the number of lines including the "COMMIT" line at the end, so it was off by one. --- pkg/proxy/iptables/proxier.go | 10 +++++----- pkg/proxy/iptables/proxier_test.go | 28 ++++++++++++---------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 74fab2f12d1..c2b19497f73 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1596,6 +1596,11 @@ func (proxier *Proxier) syncProxyRules() { "-j", "ACCEPT", ) + numberFilterIptablesRules := utilproxy.CountBytesLines(proxier.filterRules.Bytes()) + metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)).Set(float64(numberFilterIptablesRules)) + numberNatIptablesRules := utilproxy.CountBytesLines(proxier.natRules.Bytes()) + metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)).Set(float64(numberNatIptablesRules)) + // Write the end-of-table markers. utilproxy.WriteLine(proxier.filterRules, "COMMIT") utilproxy.WriteLine(proxier.natRules, "COMMIT") @@ -1608,11 +1613,6 @@ func (proxier *Proxier) syncProxyRules() { proxier.iptablesData.Write(proxier.natChains.Bytes()) proxier.iptablesData.Write(proxier.natRules.Bytes()) - numberFilterIptablesRules := utilproxy.CountBytesLines(proxier.filterRules.Bytes()) - metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)).Set(float64(numberFilterIptablesRules)) - numberNatIptablesRules := utilproxy.CountBytesLines(proxier.natRules.Bytes()) - metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)).Set(float64(numberNatIptablesRules)) - klog.V(5).InfoS("Restoring iptables", "rules", proxier.iptablesData.Bytes()) err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index f818b930b7f..9c881139754 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -913,9 +913,9 @@ COMMIT if err != nil { t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) } - // 43 here is test specific and corresponds to one more than the number of -A lines after `*nat` in `expected`. - if nNatRules != 43.0 { - t.Fatalf("Wrong number of nat rules: expected 43 received %f", nNatRules) + // 42 here is test specific and corresponds to one more than the number of -A lines after `*nat` in `expected`. + if nNatRules != 42.0 { + t.Fatalf("Wrong number of nat rules: expected 42 received %f", nNatRules) } } @@ -3308,10 +3308,9 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) { // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - // COMMIT - if nFilterRules != 7.0 { - t.Fatalf("Wrong number of filter rule: expected 7 received %f", nFilterRules) + if nFilterRules != 6.0 { + t.Fatalf("Wrong number of filter rule: expected 6 received %f", nFilterRules) } nNatRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) @@ -3319,14 +3318,13 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) { t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) } - // rules -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + // -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN // -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 // -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE // -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 // -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - // COMMIT - if nNatRules != 6.0 { - t.Fatalf("Wrong number of nat rules: expected 6 received %f", nNatRules) + if nNatRules != 5.0 { + t.Fatalf("Wrong number of nat rules: expected 5 received %f", nNatRules) } populateEndpointSlices(fp, @@ -3355,9 +3353,8 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) { // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - // COMMIT - if nFilterRules != 5.0 { - t.Fatalf("Wrong number of filter rule: expected 5 received %f", nFilterRules) + if nFilterRules != 4.0 { + t.Fatalf("Wrong number of filter rule: expected 4 received %f", nFilterRules) } nNatRules, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) if err != nil { @@ -3378,9 +3375,8 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) { // -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -s 10.0.0.5/32 -j KUBE-MARK-MASQ // -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -m udp -p udp -j DNAT --to-destination 10.0.0.5:80 // -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - // COMMIT - if nNatRules != 16.0 { - t.Fatalf("Wrong number of nat rules: expected 16 received %f", nNatRules) + if nNatRules != 15.0 { + t.Fatalf("Wrong number of nat rules: expected 15 received %f", nNatRules) } } From a1a12ca1dad3ea1d3f12603af33e5e8413885c04 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sat, 30 Oct 2021 09:14:06 -0400 Subject: [PATCH 2/8] proxy/iptables: Improve the sorting logic in TestOverallIPTablesRulesWithMultipleServices The test was sorting the iptables output so as to not depend on the order that services get processed in, but this meant it wasn't checking the relative ordering of rules (and in fact, the ordering of the rules in the "expected" string was wrong, in a way that would break things if the rules had actually been generated in that order). Add a more complicated sorting function that sorts services alphabetically while preserving the ordering of rules within each service. --- pkg/proxy/iptables/proxier_test.go | 424 ++++++++++++++++++++++++++++- 1 file changed, 414 insertions(+), 10 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 9c881139754..9cbbbf76149 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "reflect" + "regexp" "sort" "strconv" "strings" @@ -33,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/testutil" @@ -709,6 +711,414 @@ func errorf(msg string, rules []iptablestest.Rule, t *testing.T) { t.Errorf("%v", msg) } +// findAllMatches takes an array of lines and a pattern with one parenthesized group, and +// returns a sorted array of all of the unique matches of the parenthesized group. +func findAllMatches(lines []string, pattern string) []string { + regex := regexp.MustCompile(pattern) + allMatches := sets.NewString() + for _, line := range lines { + match := regex.FindStringSubmatch(line) + if len(match) == 2 { + allMatches.Insert(match[1]) + } + } + return allMatches.List() +} + +// moveMatchingLines moves lines that match pattern from input to output +func moveMatchingLines(pattern string, input, output []string) ([]string, []string) { + var newIn []string + regex := regexp.MustCompile(pattern) + for _, line := range input { + if regex.FindString(line) != "" { + output = append(output, line) + } else { + newIn = append(newIn, line) + } + } + return newIn, output +} + +// sortIPTablesRules sorts `iptables-restore` output so as to not depend on the order that +// Services get processed in, while preserving the relative ordering of related rules. +func sortIPTablesRules(ruleData string) (string, error) { + tables := strings.Split(strings.TrimPrefix(ruleData, "\n"), "COMMIT\n") + if len(tables) != 3 || tables[2] != "" { + return "", fmt.Errorf("wrong number of tables (%d) in ruleData\n%s", len(tables)-1, ruleData) + } + tables = tables[:2] + var output []string + + for _, table := range tables { + lines := strings.Split(strings.Trim(table, "\n"), "\n") + + // Move "*TABLENAME" line + lines, output = moveMatchingLines(`^\*`, lines, output) + + // findAllMatches() returns a sorted list of unique matches. So for + // each of the following, we find all the matches for the regex, then + // for each unique match (in sorted order), move all of the lines that + // contain that match. + + // Move and sort ":CHAINNAME" lines + for _, chainName := range findAllMatches(lines, `^(:[^ ]*) `) { + lines, output = moveMatchingLines(chainName, lines, output) + } + + // Move KUBE-NODEPORTS rules for each service, sorted by service name + for _, nextNodePortService := range findAllMatches(lines, `-A KUBE-NODEPORTS.*--comment "?([^ ]*)`) { + lines, output = moveMatchingLines(fmt.Sprintf(`^-A KUBE-NODEPORTS.*%s`, nextNodePortService), lines, output) + } + + // Move KUBE-SERVICES rules for each service, sorted by service name + for _, nextService := range findAllMatches(lines, `-A KUBE-SERVICES.*--comment "?([^ ]*)`) { + lines, output = moveMatchingLines(fmt.Sprintf(`^-A KUBE-SERVICES.*%s`, nextService), lines, output) + } + + // Move remaining chains, sorted by chain name + for _, nextChain := range findAllMatches(lines, `(-A KUBE-[^ ]* )`) { + lines, output = moveMatchingLines(nextChain, lines, output) + } + + // There should not be anything left, but if there is, just move it over now + // and it will show up in the diff later. + _, output = moveMatchingLines(".", lines, output) + + // The "COMMIT" line got eaten by strings.Split() above, so put it back + output = append(output, "COMMIT") + } + + // Input ended with a "\n", so make sure the output does too + output = append(output, "") + + return strings.Join(output, "\n"), nil +} + +// assertIPTablesRulesEqual asserts that the generated rules in result match the rules in +// expected, ignoring irrelevant ordering differences. +func assertIPTablesRulesEqual(t *testing.T, expected, result string) { + expected, err := sortIPTablesRules(expected) + if err != nil { + t.Fatalf("%s", err) + } + result, err = sortIPTablesRules(result) + if err != nil { + t.Fatalf("%s", err) + } + + assert.Equal(t, expected, result) +} + +func Test_sortIPTablesRules(t *testing.T) { + for _, tc := range []struct { + name string + input string + output string + error string + }{ + { + name: "basic test using each match type", + input: ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +:KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] +:KUBE-XLB-GNZBNJ2PO5MGZ6GT - [0:0] +:KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] +:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] +:KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] +:KUBE-SEP-OYPFS5VJICHGATKP - [0:0] +:KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] +:KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] +:KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 10.20.30.42/32 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT +-A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 10.20.30.42/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment ns2/svc2:p80 -j KUBE-SEP-RS4RBKLTHTF2IUXJ +-A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 +-A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 1.2.3.4/28 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT +-A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP +-A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK +-A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-X27LE4BHSL4DOUIK +-A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ +-A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -j KUBE-SEP-OYPFS5VJICHGATKP +-A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 +-A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 10.20.30.44/32 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK +-A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 10.20.30.44/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j KUBE-SEP-C6EBXVWJJZMIWKLZ +-A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 +-A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +`, + output: ` +*filter +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-SERVICES - [0:0] +-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-SEP-C6EBXVWJJZMIWKLZ - [0:0] +:KUBE-SEP-OYPFS5VJICHGATKP - [0:0] +:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +:KUBE-SEP-UKSFD7AGPMPPLUHC - [0:0] +:KUBE-SERVICES - [0:0] +:KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] +:KUBE-SVC-GNZBNJ2PO5MGZ6GT - [0:0] +:KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-XLB-GNZBNJ2PO5MGZ6GT - [0:0] +-A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT +-A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-X27LE4BHSL4DOUIK +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 10.20.30.42/32 --dport 80 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 external IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-FW-GNZBNJ2PO5MGZ6GT +-A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK +-A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 10.20.30.44/32 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK +-A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 -j KUBE-SVC-4SW47YFZTEDKD3PK +-A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 1.2.3.4/28 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT +-A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -s 10.180.0.5/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-C6EBXVWJJZMIWKLZ -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.5:80 +-A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -s 10.180.0.3/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-OYPFS5VJICHGATKP -m comment --comment ns3/svc3:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 +-A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -s 10.180.0.2/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -s 10.180.0.4/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-UKSFD7AGPMPPLUHC -m comment --comment ns4/svc4:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.4:80 +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 cluster IP" -m tcp -p tcp -d 10.20.30.44/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment "ns4/svc4:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC +-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j KUBE-SEP-C6EBXVWJJZMIWKLZ +-A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 cluster IP" -m tcp -p tcp -d 10.20.30.42/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-GNZBNJ2PO5MGZ6GT -m comment --comment ns2/svc2:p80 -j KUBE-SEP-RS4RBKLTHTF2IUXJ +-A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ +-A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment ns3/svc3:p80 -j KUBE-SEP-OYPFS5VJICHGATKP +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT +-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP +COMMIT +`, + }, + { + name: "not enough tables", + input: ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +`, + error: "wrong number of tables (1)", + }, + { + name: "too many tables", + input: ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*mangle +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +`, + error: "wrong number of tables (3)", + }, + { + name: "correctly match same service name in different styles of comments", + input: ` +*filter +COMMIT +*nat +:KUBE-SERVICES - [0:0] +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 +-A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 +-A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" svc1 line 1 +-A KUBE-SERVICES -m comment --comment ns1/svc1 svc1 line 2 +-A KUBE-SERVICES -m comment --comment "ns1/svc1 blah" svc1 line 3 +-A KUBE-SERVICES -m comment --comment ns4/svc4 svc4 line 1 +-A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" svc4 line 2 +-A KUBE-SERVICES -m comment --comment "ns4/svc4 blah" svc4 line 3 +-A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" svc3 line 1 +-A KUBE-SERVICES -m comment --comment "ns3/svc3 blah" svc3 line 2 +-A KUBE-SERVICES -m comment --comment ns3/svc3 svc3 line 3 +COMMIT +`, + output: ` +*filter +COMMIT +*nat +:KUBE-SERVICES - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" svc1 line 1 +-A KUBE-SERVICES -m comment --comment ns1/svc1 svc1 line 2 +-A KUBE-SERVICES -m comment --comment "ns1/svc1 blah" svc1 line 3 +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 +-A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 +-A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 +-A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" svc3 line 1 +-A KUBE-SERVICES -m comment --comment "ns3/svc3 blah" svc3 line 2 +-A KUBE-SERVICES -m comment --comment ns3/svc3 svc3 line 3 +-A KUBE-SERVICES -m comment --comment ns4/svc4 svc4 line 1 +-A KUBE-SERVICES -m comment --comment "ns4/svc4:p80 cluster IP" svc4 line 2 +-A KUBE-SERVICES -m comment --comment "ns4/svc4 blah" svc4 line 3 +COMMIT +`, + }, + { + name: "unexpected junk lines are preserved", + input: ` +*filter +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] +:KUBE-AAAAA - [0:0] +:KUBE-ZZZZZ - [0:0] +:WHY-IS-THIS-CHAIN-HERE - [0:0] +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 +-A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 +-A KUBE-ZZZZZ -m comment --comment "mystery chain number 1" +-A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 +-A WHY-IS-THIS-CHAIN-HERE -j ACCEPT +-A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 +-A KUBE-AAAAA -m comment --comment "mystery chain number 2" +COMMIT +`, + output: ` +*filter +COMMIT +*nat +:KUBE-AAAAA - [0:0] +:KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] +:KUBE-SERVICES - [0:0] +:KUBE-ZZZZZ - [0:0] +:WHY-IS-THIS-CHAIN-HERE - [0:0] +-A KUBE-SERVICES -m comment --comment "ns2/svc2:p80 cluster IP" svc2 line 1 +-A KUBE-SERVICES -m comment --comment ns2/svc2 svc2 line 2 +-A KUBE-SERVICES -m comment --comment "ns2/svc2 blah" svc2 line 3 +-A KUBE-AAAAA -m comment --comment "mystery chain number 2" +-A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 +-A KUBE-ZZZZZ -m comment --comment "mystery chain number 1" +-A WHY-IS-THIS-CHAIN-HERE -j ACCEPT +COMMIT +`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + out, err := sortIPTablesRules(tc.input) + if err == nil { + if tc.error != "" { + t.Errorf("unexpectedly did not get error") + } else { + assert.Equal(t, strings.TrimPrefix(tc.output, "\n"), out) + } + } else { + if tc.error == "" { + t.Errorf("got unexpected error: %v", err) + } else if !strings.HasPrefix(err.Error(), tc.error) { + t.Errorf("got wrong error: %v (expected %q)", err, tc.error) + } + } + }) + } +} + // TestOverallIPTablesRulesWithMultipleServices creates 4 types of services: ClusterIP, // LoadBalancer, ExternalIP and NodePort and verifies if the NAT table rules created // are exactly the same as what is expected. This test provides an overall view of how @@ -829,7 +1239,8 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { fp.syncProxyRules() - expected := `*filter + expected := ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -874,8 +1285,8 @@ COMMIT -A KUBE-SEP-RS4RBKLTHTF2IUXJ -m comment --comment ns2/svc2:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.2:80 -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -s 1.2.3.4/28 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT -A KUBE-FW-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT -A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns2/svc2:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-GNZBNJ2PO5MGZ6GT -A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT @@ -900,14 +1311,7 @@ COMMIT -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS COMMIT ` - // Adding logic to split and sort the strings because - // depending on the creation order of services, the user-chains get jumbled. - expectedSlice := strings.Split(strings.TrimSuffix(expected, "\n"), "\n") - sort.Strings(expectedSlice) - originalSlice := strings.Split(strings.TrimSuffix(fp.iptablesData.String(), "\n"), "\n") - sort.Strings(originalSlice) - - assert.Equal(t, expectedSlice, originalSlice) + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) nNatRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) if err != nil { From 9403bfb17893462abb90b02206296688c0e3de3b Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Mon, 25 Oct 2021 18:09:38 -0400 Subject: [PATCH 3/8] proxy/iptables: Misc improvements to unit test The original tests here were very shy about looking at the iptables output, and just relied on checks like "make sure there's a jump to table X that also includes string Y somewhere in it" and stuff like that. Whereas the newer tests were just like, "eh, here's a wall of text, make sure the iptables output is exactly that". Although the latter looks messier in the code, it's more precise, and it's easier to update correctly when you change the rules. So just make all of the tests do a check on the full iptables output. (Note that I didn't double-check any of the output; I'm just assuming that the output of the current iptables proxy code is actually correct...) Also, don't hardcode the expected number of rules in the metrics tests, so that there's one less thing to adjust when rules change. Also, use t.Run() in one place to get more precise errors on failure. --- pkg/proxy/iptables/proxier_test.go | 1105 ++++++++++++++++------------ 1 file changed, 622 insertions(+), 483 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 9cbbbf76149..ab7dabc7d89 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -521,194 +521,21 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { return p } -func hasSessionAffinityRule(rules []iptablestest.Rule) bool { - for _, r := range rules { - if _, ok := r[iptablestest.Recent]; ok { - return true - } - } - return false -} - -func hasJump(rules []iptablestest.Rule, destChain, destIP string, destPort int) bool { - destPortStr := strconv.Itoa(destPort) - match := false - for _, r := range rules { - if r[iptablestest.Jump] == destChain { - match = true - if destIP != "" { - if strings.Contains(r[iptablestest.Destination], destIP) && (strings.Contains(r[iptablestest.DPort], destPortStr) || r[iptablestest.DPort] == "") { - return true - } - match = false - } - if destPort != 0 { - if strings.Contains(r[iptablestest.DPort], destPortStr) && (strings.Contains(r[iptablestest.Destination], destIP) || r[iptablestest.Destination] == "") { - return true - } - match = false - } - } - } - return match -} - -func hasSrcType(rules []iptablestest.Rule, srcType string) bool { - for _, r := range rules { - if r[iptablestest.SrcType] != srcType { +func countRules(table, data string) int { + inRightTable := false + rules := 0 + for _, line := range strings.Split(data, "\n") { + if line == "" { continue } - - return true - } - - return false -} - -func hasMasqRandomFully(rules []iptablestest.Rule) bool { - for _, r := range rules { - if r[iptablestest.Masquerade] == "--random-fully" { - return true + if line[0] == '*' { + inRightTable = (line == "*"+table) + } + if inRightTable && line[0] == '-' { + rules++ } } - return false -} - -func TestHasJump(t *testing.T) { - testCases := map[string]struct { - rules []iptablestest.Rule - destChain string - destIP string - destPort int - expected bool - }{ - "case 1": { - // Match the 1st rule(both dest IP and dest Port) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "--dport ": "80", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "KUBE-MARK-MASQ"}, - }, - destChain: "REJECT", - destIP: "10.20.30.41", - destPort: 80, - expected: true, - }, - "case 2": { - // Match the 2nd rule(dest Port) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "", - destPort: 3001, - expected: true, - }, - "case 3": { - // Match both dest IP and dest Port - rules: []iptablestest.Rule{ - {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, - }, - destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", - destIP: "1.2.3.4", - destPort: 80, - expected: true, - }, - "case 4": { - // Match dest IP but doesn't match dest Port - rules: []iptablestest.Rule{ - {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, - }, - destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", - destIP: "1.2.3.4", - destPort: 8080, - expected: false, - }, - "case 5": { - // Match dest Port but doesn't match dest IP - rules: []iptablestest.Rule{ - {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, - }, - destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", - destIP: "10.20.30.40", - destPort: 80, - expected: false, - }, - "case 6": { - // Match the 2nd rule(dest IP) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"-d ": "1.2.3.4/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "1.2.3.4", - destPort: 8080, - expected: true, - }, - "case 7": { - // Match the 2nd rule(dest Port) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "1.2.3.4", - destPort: 3001, - expected: true, - }, - "case 8": { - // Match the 1st rule(dest IP) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "10.20.30.41", - destPort: 8080, - expected: true, - }, - "case 9": { - rules: []iptablestest.Rule{ - {"-j ": "KUBE-SEP-LWSOSDSHMKPJHHJV"}, - }, - destChain: "KUBE-SEP-LWSOSDSHMKPJHHJV", - destIP: "", - destPort: 0, - expected: true, - }, - "case 10": { - rules: []iptablestest.Rule{ - {"-j ": "KUBE-SEP-FOO"}, - }, - destChain: "KUBE-SEP-BAR", - destIP: "", - destPort: 0, - expected: false, - }, - } - - for k, tc := range testCases { - if got := hasJump(tc.rules, tc.destChain, tc.destIP, tc.destPort); got != tc.expected { - t.Errorf("%v: expected %v, got %v", k, tc.expected, got) - } - } -} - -func hasDNAT(rules []iptablestest.Rule, endpoint string) bool { - for _, r := range rules { - if r[iptablestest.ToDest] == endpoint { - return true - } - } - return false -} - -func errorf(msg string, rules []iptablestest.Rule, t *testing.T) { - for _, r := range rules { - t.Logf("%q", r) - } - t.Errorf("%v", msg) + return rules } // findAllMatches takes an array of lines and a pattern with one parenthesized group, and @@ -780,6 +607,11 @@ func sortIPTablesRules(ruleData string) (string, error) { lines, output = moveMatchingLines(nextChain, lines, output) } + // Some tests have deletions... + for _, nextChain := range findAllMatches(lines, `(-X KUBE-.*)`) { + lines, output = moveMatchingLines(nextChain, lines, output) + } + // There should not be anything left, but if there is, just move it over now // and it will show up in the diff later. _, output = moveMatchingLines(".", lines, output) @@ -809,6 +641,21 @@ func assertIPTablesRulesEqual(t *testing.T, expected, result string) { assert.Equal(t, expected, result) } +// assertIPTablesRulesNotEqual asserts that the generated rules in result DON'T match the +// rules in expected, ignoring irrelevant ordering differences. +func assertIPTablesRulesNotEqual(t *testing.T, expected, result string) { + expected, err := sortIPTablesRules(expected) + if err != nil { + t.Fatalf("%s", err) + } + result, err = sortIPTablesRules(result) + if err != nil { + t.Fatalf("%s", err) + } + + assert.NotEqual(t, expected, result) +} + func Test_sortIPTablesRules(t *testing.T) { for _, tc := range []struct { name string @@ -1313,13 +1160,16 @@ COMMIT ` assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) - nNatRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) + natRulesMetric, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) if err != nil { t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) } - // 42 here is test specific and corresponds to one more than the number of -A lines after `*nat` in `expected`. - if nNatRules != 42.0 { - t.Fatalf("Wrong number of nat rules: expected 42 received %f", nNatRules) + nNatRules := int(natRulesMetric) + + expectedNatRules := countRules("nat", fp.iptablesData.String()) + + if nNatRules != expectedNatRules { + t.Fatalf("Wrong number of nat rules: expected %d received %d", expectedNatRules, nNatRules) } } @@ -1345,15 +1195,32 @@ func TestClusterIPReject(t *testing.T) { ) fp.syncProxyRules() - svcChain := string(servicePortChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)))) - svcRules := ipt.GetRules(svcChain) - if len(svcRules) != 0 { - errorf(fmt.Sprintf("Unexpected rule for chain %v service %v without endpoints", svcChain, svcPortName), svcRules, t) - } - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, iptablestest.Reject, svcIP, svcPort) { - errorf(fmt.Sprintf("Failed to find a %v rule for service %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) - } + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j REJECT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` + + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestClusterIPEndpointsJump(t *testing.T) { @@ -1396,27 +1263,38 @@ func TestClusterIPEndpointsJump(t *testing.T) { fp.syncProxyRules() - epStr := fmt.Sprintf("%s:%d", epIP, svcPort) - svcChain := string(servicePortChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)))) - epChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)), epStr)) + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, svcChain, svcIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump from KUBE-SERVICES to %v chain", svcChain), kubeSvcRules, t) - } - - svcRules := ipt.GetRules(svcChain) - if !hasJump(svcRules, string(KubeMarkMasqChain), svcIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump from %v to KUBE-MARK-MASQ chain", svcChain), svcRules, t) - } - - if !hasJump(svcRules, epChain, "", 0) { - errorf(fmt.Sprintf("Failed to jump to ep chain %v", epChain), svcRules, t) - } - epRules := ipt.GetRules(epChain) - if !hasDNAT(epRules, epStr) { - errorf(fmt.Sprintf("Endpoint chain %v lacks DNAT to %v", epChain, epStr), epRules, t) - } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestLoadBalancer(t *testing.T) { @@ -1470,19 +1348,47 @@ func TestLoadBalancer(t *testing.T) { fp.syncProxyRules() - proto := strings.ToLower(string(v1.ProtocolTCP)) - fwChain := string(serviceFirewallChainName(svcPortName.String(), proto)) - svcChain := string(servicePortChainName(svcPortName.String(), proto)) + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-FW-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-MASQ +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -s 1.2.3.4/28 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, fwChain, svcLBIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump to firewall chain %v", fwChain), kubeSvcRules, t) - } - - fwRules := ipt.GetRules(fwChain) - if !hasJump(fwRules, svcChain, "", 0) || !hasJump(fwRules, string(KubeMarkMasqChain), "", 0) { - errorf(fmt.Sprintf("Failed to find jump from firewall chain %v to svc chain %v", fwChain, svcChain), fwRules, t) - } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestNodePort(t *testing.T) { @@ -1536,18 +1442,40 @@ func TestNodePort(t *testing.T) { fp.syncProxyRules() - proto := strings.ToLower(string(v1.ProtocolTCP)) - svcChain := string(servicePortChainName(svcPortName.String(), proto)) + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` - kubeNodePortRules := ipt.GetRules(string(kubeNodePortsChain)) - if !hasJump(kubeNodePortRules, svcChain, "", svcNodePort) { - errorf(fmt.Sprintf("Failed to find jump to svc chain %v", svcChain), kubeNodePortRules, t) - } - expectedNodePortNonLocalTrafficMasqueradeRule := `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ` - svcRules := ipt.GetRules(svcChain) - if !strings.Contains(fp.iptablesData.String(), expectedNodePortNonLocalTrafficMasqueradeRule) { - errorf(fmt.Sprintf("Didn't find the masquerade rule for node port non-local traffic in svc chain %v", svcChain), svcRules, t) - } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestHealthCheckNodePort(t *testing.T) { @@ -1588,10 +1516,35 @@ func TestHealthCheckNodePort(t *testing.T) { fp.syncProxyRules() - kubeNodePortsRules := ipt.GetRules(string(kubeNodePortsChain)) - if !hasJump(kubeNodePortsRules, iptablestest.Accept, "", svcHealthCheckNodePort) { - errorf(fmt.Sprintf("Failed to find Accept rule"), kubeNodePortsRules, t) - } + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 10.20.30.42/32 --dport 80 -j REJECT +-A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT +-A KUBE-NODEPORTS -m comment --comment "ns1/svc1:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 127.0.0.1 -j KUBE-NODEPORTS +COMMIT +` + + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestMasqueradeRule(t *testing.T) { @@ -1600,14 +1553,36 @@ func TestMasqueradeRule(t *testing.T) { fp := NewFakeProxier(ipt) fp.syncProxyRules() - postRoutingRules := ipt.GetRules(string(kubePostroutingChain)) - if !hasJump(postRoutingRules, "MASQUERADE", "", 0) { - errorf(fmt.Sprintf("Failed to find -j MASQUERADE in %s chain", kubePostroutingChain), postRoutingRules, t) - } - if hasMasqRandomFully(postRoutingRules) != testcase { - probs := map[bool]string{false: "found", true: "did not find"} - errorf(fmt.Sprintf("%s --random-fully in -j MASQUERADE rule in %s chain when HasRandomFully()==%v", probs[testcase], kubePostroutingChain, testcase), postRoutingRules, t) + expectedFmt := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE%s +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` + var expected string + if testcase { + expected = fmt.Sprintf(expectedFmt, " --random-fully") + } else { + expected = fmt.Sprintf(expectedFmt, "") } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } } @@ -1638,10 +1613,33 @@ func TestExternalIPsReject(t *testing.T) { fp.syncProxyRules() - kubeSvcRules := ipt.GetRules(string(kubeExternalServicesChain)) - if !hasJump(kubeSvcRules, iptablestest.Reject, svcExternalIPs, svcPort) { - errorf(fmt.Sprintf("Failed to find a %v rule for externalIP %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) - } + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j REJECT +-A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 -j REJECT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` + + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestOnlyLocalExternalIPs(t *testing.T) { @@ -1671,8 +1669,6 @@ func TestOnlyLocalExternalIPs(t *testing.T) { ) epIP1 := "10.180.0.1" epIP2 := "10.180.2.1" - epStrLocal := fmt.Sprintf("%s:%d", epIP1, svcPort) - epStrNonLocal := fmt.Sprintf("%s:%d", epIP2, svcPort) tcpProtocol := v1.ProtocolTCP populateEndpointSlices(fp, makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { @@ -1693,24 +1689,48 @@ func TestOnlyLocalExternalIPs(t *testing.T) { fp.syncProxyRules() - proto := strings.ToLower(string(v1.ProtocolTCP)) - lbChain := string(serviceLBChainName(svcPortName.String(), proto)) + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 -j KUBE-XLB-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Balancing rule 0 for ns1/svc1:p80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` - nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)), epStrLocal)) - localEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)), epStrNonLocal)) - - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, lbChain, svcExternalIPs, svcPort) { - errorf(fmt.Sprintf("Failed to find jump to xlb chain %v", lbChain), kubeSvcRules, t) - } - - lbRules := ipt.GetRules(lbChain) - if hasJump(lbRules, nonLocalEpChain, "", 0) { - errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) - } - if !hasJump(lbRules, localEpChain, "", 0) { - errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrNonLocal), lbRules, t) - } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } // TestNonLocalExternalIPs tests if we add the masquerade rule into svcChain in order to @@ -1760,32 +1780,45 @@ func TestNonLocalExternalIPs(t *testing.T) { ) fp.syncProxyRules() - proto := strings.ToLower(string(v1.ProtocolTCP)) - lbChain := string(serviceLBChainName(svcPortName.String(), proto)) - svcChain := string(servicePortChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)))) - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, svcChain, svcExternalIPs, svcPort) { - errorf(fmt.Sprintf("Failed to find jump to svc chain %v", svcChain), kubeSvcRules, t) - } + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` - svcRules := ipt.GetRules(svcChain) - if len(svcRules) != 4 { - t.Errorf("expected svcChain %v to have 4 rules, got %v", svcChain, len(svcRules)) - } - if !hasJump(svcRules, string(KubeMarkMasqChain), svcIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump from %v to KUBE-MARK-MASQ chain", svcChain), svcRules, t) - } - expectedExternalIPNonLocalTrafficMasqueradeRule := `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 external IP" -m tcp -p tcp -d 50.60.70.81/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ` - - if !strings.Contains(fp.iptablesData.String(), expectedExternalIPNonLocalTrafficMasqueradeRule) { - errorf(fmt.Sprintf("Didn't find the masquerade rule for external-ip non-local traffic in svc chain %v", svcChain), svcRules, t) - } - - lbRules := ipt.GetRules(lbChain) - if len(lbRules) != 0 { - t.Errorf("expected svclbChain %v to have 0 rules, got %v", lbChain, len(lbRules)) - } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestNodePortReject(t *testing.T) { @@ -1814,10 +1847,33 @@ func TestNodePortReject(t *testing.T) { fp.syncProxyRules() - kubeSvcRules := ipt.GetRules(string(kubeExternalServicesChain)) - if !hasJump(kubeSvcRules, iptablestest.Reject, svcIP, svcNodePort) { - errorf(fmt.Sprintf("Failed to find a %v rule for service %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) - } + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j REJECT +-A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` + + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestLoadBalancerReject(t *testing.T) { @@ -1856,16 +1912,35 @@ func TestLoadBalancerReject(t *testing.T) { fp.syncProxyRules() - kubeSvcExtRules := ipt.GetRules(string(kubeExternalServicesChain)) - if !hasJump(kubeSvcExtRules, iptablestest.Reject, svcLBIP, svcPort) { - errorf(fmt.Sprintf("Failed to find a %v rule for LoadBalancer %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcExtRules, t) - } - - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if hasJump(kubeSvcRules, iptablestest.Reject, svcLBIP, svcPort) { - errorf(fmt.Sprintf("Found a %v rule for LoadBalancer %v with no endpoints in kubeServicesChain", iptablestest.Reject, svcPortName), kubeSvcRules, t) - } + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j REJECT +-A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j REJECT +-A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m tcp -p tcp --dport 3001 -j REJECT +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestOnlyLocalLoadBalancing(t *testing.T) { @@ -1905,8 +1980,6 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { epIP1 := "10.180.0.1" epIP2 := "10.180.2.1" - epStrLocal := fmt.Sprintf("%s:%d", epIP1, svcPort) - epStrNonLocal := fmt.Sprintf("%s:%d", epIP2, svcPort) tcpProtocol := v1.ProtocolTCP populateEndpointSlices(fp, makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { @@ -1927,51 +2000,155 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { fp.syncProxyRules() - proto := strings.ToLower(string(v1.ProtocolTCP)) - fwChain := string(serviceFirewallChainName(svcPortName.String(), proto)) - lbChain := string(serviceLBChainName(svcPortName.String(), proto)) + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] +:KUBE-FW-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-XLB-XPGD46QRK7WJZT7O +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --set -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-ZX7GRIZKSNUQ3LAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Balancing rule 0 for ns1/svc1:p80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` - nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)), epStrLocal)) - localEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)), epStrNonLocal)) - - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, fwChain, svcLBIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump to firewall chain %v", fwChain), kubeSvcRules, t) - } - - fwRules := ipt.GetRules(fwChain) - if !hasJump(fwRules, lbChain, "", 0) { - errorf(fmt.Sprintf("Failed to find jump from firewall chain %v to svc chain %v", fwChain, lbChain), fwRules, t) - } - if hasJump(fwRules, string(KubeMarkMasqChain), "", 0) { - errorf(fmt.Sprintf("Found jump from fw chain %v to MASQUERADE", fwChain), fwRules, t) - } - - lbRules := ipt.GetRules(lbChain) - if hasJump(lbRules, nonLocalEpChain, "", 0) { - errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) - } - if !hasJump(lbRules, localEpChain, "", 0) { - errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrNonLocal), lbRules, t) - } - if !hasSessionAffinityRule(lbRules) { - errorf(fmt.Sprintf("Didn't find session affinity rule from lb chain %v", lbChain), lbRules, t) - } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) - onlyLocalNodePorts(t, fp, ipt) + + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Balancing rule 0 for ns1/svc1:p80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 10.20.30.51 -j KUBE-NODEPORTS +COMMIT +` + onlyLocalNodePorts(t, fp, ipt, expected) } func TestOnlyLocalNodePorts(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) - onlyLocalNodePorts(t, fp, ipt) + + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-XLB-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +:KUBE-SEP-ZX7GRIZKSNUQ3LAJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Balancing rule 0 for ns1/svc1:p80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -d 10.20.30.51 -j KUBE-NODEPORTS +COMMIT +` + onlyLocalNodePorts(t, fp, ipt, expected) } -func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTables) { +func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTables, expected string) { svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 @@ -1997,8 +2174,6 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable epIP1 := "10.180.0.1" epIP2 := "10.180.2.1" - epStrLocal := fmt.Sprintf("%s:%d", epIP1, svcPort) - epStrNonLocal := fmt.Sprintf("%s:%d", epIP2, svcPort) tcpProtocol := v1.ProtocolTCP populateEndpointSlices(fp, makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { @@ -2025,36 +2200,7 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable fp.syncProxyRules() - proto := strings.ToLower(string(v1.ProtocolTCP)) - lbChain := string(serviceLBChainName(svcPortName.String(), proto)) - - nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), proto, epStrLocal)) - localEpChain := string(servicePortEndpointChainName(svcPortName.String(), proto, epStrNonLocal)) - - kubeNodePortRules := ipt.GetRules(string(kubeNodePortsChain)) - if !hasJump(kubeNodePortRules, lbChain, "", svcNodePort) { - errorf(fmt.Sprintf("Failed to find jump to lb chain %v", lbChain), kubeNodePortRules, t) - } - if !hasJump(kubeNodePortRules, string(KubeMarkMasqChain), "", svcNodePort) { - errorf(fmt.Sprintf("Failed to find jump to %s chain for destination IP %d", KubeMarkMasqChain, svcNodePort), kubeNodePortRules, t) - } - - kubeServiceRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeServiceRules, string(kubeNodePortsChain), "10.20.30.51", 0) { - errorf(fmt.Sprintf("Failed to find jump to KUBE-NODEPORTS chain %v", string(kubeNodePortsChain)), kubeServiceRules, t) - } - - svcChain := string(servicePortChainName(svcPortName.String(), proto)) - lbRules := ipt.GetRules(lbChain) - if hasJump(lbRules, nonLocalEpChain, "", 0) { - errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) - } - if !hasJump(lbRules, svcChain, "", 0) || !hasSrcType(lbRules, "LOCAL") { - errorf(fmt.Sprintf("Did not find jump from lb chain %v to svc %v with src-type LOCAL", lbChain, svcChain), lbRules, t) - } - if !hasJump(lbRules, localEpChain, "", 0) { - errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrLocal), lbRules, t) - } + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) } func TestComputeProbability(t *testing.T) { @@ -3243,7 +3389,8 @@ func Test_updateEndpointsMap(t *testing.T) { // the shared EndpointChangeTracker and EndpointSliceCache. This test ensures that the // iptables proxier supports translating EndpointSlices to iptables output. func TestEndpointSliceE2E(t *testing.T) { - expectedIPTablesWithSlice := `*filter + expectedIPTablesWithSlice := ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -3335,15 +3482,16 @@ COMMIT fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - assert.Equal(t, expectedIPTablesWithSlice, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String()) fp.OnEndpointSliceDelete(endpointSlice) fp.syncProxyRules() - assert.NotEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String()) } func TestHealthCheckNodePortE2E(t *testing.T) { - expectedIPTables := `*filter + expectedIPTables := ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -3446,11 +3594,11 @@ COMMIT } fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - assert.Equal(t, expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, expectedIPTables, fp.iptablesData.String()) fp.OnServiceDelete(svc) fp.syncProxyRules() - assert.NotEqual(t, expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, expectedIPTables, fp.iptablesData.String()) } // Test_HealthCheckNodePortWhenTerminating tests that health check node ports are not enabled when all local endpoints are terminating @@ -3701,34 +3849,28 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) { }), ) fp.syncProxyRules() + iptablesData := fp.iptablesData.String() - nFilterRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter))) + nFilterRulesMetric, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter))) if err != nil { t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) } - // -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m udp -p udp -d 10.20.30.41/32 --dport 80 -j REJECT - // -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m udp -p udp --dport 31201 -j REJECT - // -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + nFilterRules := int(nFilterRulesMetric) + expectedFilterRules := countRules("filter", iptablesData) - if nFilterRules != 6.0 { - t.Fatalf("Wrong number of filter rule: expected 6 received %f", nFilterRules) + if nFilterRules != expectedFilterRules { + t.Fatalf("Wrong number of filter rule: expected %d got %d\n%s", expectedFilterRules, nFilterRules, iptablesData) } - nNatRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) + nNatRulesMetric, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) if err != nil { t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) } + nNatRules := int(nNatRulesMetric) + expectedNatRules := countRules("nat", iptablesData) - // -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - // -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - // -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - // -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - // -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - if nNatRules != 5.0 { - t.Fatalf("Wrong number of nat rules: expected 5 received %f", nNatRules) + if nNatRules != expectedNatRules { + t.Fatalf("Wrong number of nat rules: expected %d got %d\n%s", expectedNatRules, nNatRules, iptablesData) } populateEndpointSlices(fp, @@ -3748,39 +3890,28 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) { ) fp.syncProxyRules() + iptablesData = fp.iptablesData.String() - nFilterRules, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter))) + nFilterRulesMetric, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter))) if err != nil { t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) } - // -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - if nFilterRules != 4.0 { - t.Fatalf("Wrong number of filter rule: expected 4 received %f", nFilterRules) + nFilterRules = int(nFilterRulesMetric) + expectedFilterRules = countRules("filter", iptablesData) + + if nFilterRules != expectedFilterRules { + t.Fatalf("Wrong number of filter rule: expected %d got %d\n%s", expectedFilterRules, nFilterRules, iptablesData) } - nNatRules, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) + + nNatRulesMetric, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) if err != nil { t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) } - // -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - // -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - // -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - // -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - // -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m udp -p udp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ - // -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m udp -p udp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-OJWW7NSBVZTDHXNW - // -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m udp -p udp --dport 31201 -j KUBE-MARK-MASQ - // -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m udp -p udp --dport 31201 -j KUBE-SVC-OJWW7NSBVZTDHXNW - // -A KUBE-SVC-OJWW7NSBVZTDHXNW -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-AMT2SNW3YUNHJFJG - // -A KUBE-SEP-AMT2SNW3YUNHJFJG -m comment --comment ns1/svc1:p80 -s 10.0.0.2/32 -j KUBE-MARK-MASQ - // -A KUBE-SEP-AMT2SNW3YUNHJFJG -m comment --comment ns1/svc1:p80 -m udp -p udp -j DNAT --to-destination 10.0.0.2:80 - // -A KUBE-SVC-OJWW7NSBVZTDHXNW -m comment --comment ns1/svc1:p80 -j KUBE-SEP-OUFLBLJVR33W4FIZ - // -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -s 10.0.0.5/32 -j KUBE-MARK-MASQ - // -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -m udp -p udp -j DNAT --to-destination 10.0.0.5:80 - // -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - if nNatRules != 15.0 { - t.Fatalf("Wrong number of nat rules: expected 15 received %f", nNatRules) + nNatRules = int(nNatRulesMetric) + expectedNatRules = countRules("nat", iptablesData) + + if nNatRules != expectedNatRules { + t.Fatalf("Wrong number of nat rules: expected %d got %d\n%s", expectedNatRules, nNatRules, iptablesData) } } @@ -3797,7 +3928,8 @@ func TestInternalTrafficPolicyE2E(t *testing.T) { cluster := v1.ServiceInternalTrafficPolicyCluster local := v1.ServiceInternalTrafficPolicyLocal - clusterExpectedIPTables := `*filter + clusterExpectedIPTables := ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -3865,7 +3997,8 @@ COMMIT {"10.0.1.3", "host2"}, }, expectEndpointRule: true, - expectedIPTablesWithSlice: `*filter + expectedIPTablesWithSlice: ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -3905,7 +4038,8 @@ COMMIT {"10.0.1.3", "host2"}, }, expectEndpointRule: false, - expectedIPTablesWithSlice: `*filter + expectedIPTablesWithSlice: ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -3944,60 +4078,62 @@ COMMIT } for _, tc := range testCases { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, tc.featureGateOn)() - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - fp.OnServiceSynced() - fp.OnEndpointSlicesSynced() + t.Run(tc.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, tc.featureGateOn)() + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + fp.OnServiceSynced() + fp.OnEndpointSlicesSynced() - serviceName := "svc1" - namespaceName := "ns1" + serviceName := "svc1" + namespaceName := "ns1" - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, - Spec: v1.ServiceSpec{ - ClusterIP: "172.20.1.1", - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{{Name: "", Port: 80, Protocol: v1.ProtocolTCP}}, - }, - } - if tc.internalTrafficPolicy != nil { - svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy - } + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, + Spec: v1.ServiceSpec{ + ClusterIP: "172.20.1.1", + Selector: map[string]string{"foo": "bar"}, + Ports: []v1.ServicePort{{Name: "", Port: 80, Protocol: v1.ProtocolTCP}}, + }, + } + if tc.internalTrafficPolicy != nil { + svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy + } - fp.OnServiceAdd(svc) + fp.OnServiceAdd(svc) - tcpProtocol := v1.ProtocolTCP - endpointSlice := &discovery.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-1", serviceName), - Namespace: namespaceName, - Labels: map[string]string{discovery.LabelServiceName: serviceName}, - }, - Ports: []discovery.EndpointPort{{ - Name: utilpointer.StringPtr(""), - Port: utilpointer.Int32Ptr(80), - Protocol: &tcpProtocol, - }}, - AddressType: discovery.AddressTypeIPv4, - } - for _, ep := range tc.endpoints { - endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{ - Addresses: []string{ep.ip}, - Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, - NodeName: utilpointer.StringPtr(ep.hostname), - }) - } + tcpProtocol := v1.ProtocolTCP + endpointSlice := &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-1", serviceName), + Namespace: namespaceName, + Labels: map[string]string{discovery.LabelServiceName: serviceName}, + }, + Ports: []discovery.EndpointPort{{ + Name: utilpointer.StringPtr(""), + Port: utilpointer.Int32Ptr(80), + Protocol: &tcpProtocol, + }}, + AddressType: discovery.AddressTypeIPv4, + } + for _, ep := range tc.endpoints { + endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{ + Addresses: []string{ep.ip}, + Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, + NodeName: utilpointer.StringPtr(ep.hostname), + }) + } - fp.OnEndpointSliceAdd(endpointSlice) - fp.syncProxyRules() - assert.Equal(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) - - if tc.expectEndpointRule { - fp.OnEndpointSliceDelete(endpointSlice) + fp.OnEndpointSliceAdd(endpointSlice) fp.syncProxyRules() - assert.NotEqual(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) - } + assertIPTablesRulesEqual(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) + + if tc.expectEndpointRule { + fp.OnEndpointSliceDelete(endpointSlice) + fp.syncProxyRules() + assertIPTablesRulesNotEqual(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String()) + } + }) } } @@ -4095,7 +4231,8 @@ func Test_EndpointSliceWithTerminatingEndpoints(t *testing.T) { }, }, }, - expectedIPTables: `*filter + expectedIPTables: ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -4228,7 +4365,8 @@ COMMIT }, }, }, - expectedIPTables: `*filter + expectedIPTables: ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -4353,7 +4491,8 @@ COMMIT }, }, }, - expectedIPTables: `*filter + expectedIPTables: ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -4478,7 +4617,8 @@ COMMIT }, }, }, - expectedIPTables: `*filter + expectedIPTables: ` +*filter :KUBE-SERVICES - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FORWARD - [0:0] @@ -4529,7 +4669,6 @@ COMMIT for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProxyTerminatingEndpoints, testcase.terminatingFeatureGate)() ipt := iptablestest.NewFake() @@ -4542,11 +4681,11 @@ COMMIT fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() t.Log(fp.iptablesData.String()) - assert.Equal(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() - assert.NotEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + assertIPTablesRulesNotEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) }) } } From 799c222c8415916d4de20352db7e5ac793af4775 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 3 Nov 2021 10:03:01 -0400 Subject: [PATCH 4/8] proxy/iptables: test that we create a consistent set of iptables rules --- pkg/proxy/iptables/proxier_test.go | 81 ++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index ab7dabc7d89..d21c5e76423 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -566,6 +566,43 @@ func moveMatchingLines(pattern string, input, output []string) ([]string, []stri return newIn, output } +// assertIPTablesRuleJumps asserts that every `-j` in the given rules jumps to a chain +// that we created and added rules to +func assertIPTablesRuleJumps(lines []string) error { + tableName := lines[0] + + createdChains := sets.NewString(findAllMatches(lines, `^:([^ ]*)`)...) + filledChains := sets.NewString(findAllMatches(lines, `-A ([^ ]*)`)...) + + jumpedChains := sets.NewString(findAllMatches(lines, `-j ([^ ]*)`)...) + // Ignore jumps to built-in chains + jumpedChains.Delete("ACCEPT", "REJECT", "DROP", "MARK", "RETURN", "DNAT", "SNAT", "MASQUERADE") + // KubeMarkDropChain is created by kubelet, not kube-proxy + jumpedChains.Delete(string(KubeMarkDropChain)) + // In some cases it's not a bug if we jump to a chain when that chain is empty + jumpedChains.Delete(string(kubeNodePortsChain)) + + missingChains := jumpedChains.Difference(createdChains) + missingChains = missingChains.Union(filledChains.Difference(createdChains)) + if len(missingChains) > 0 { + return fmt.Errorf("some chains in %s are used but were not created: %v", tableName, missingChains.List()) + } + + emptyChains := jumpedChains.Difference(filledChains) + if len(emptyChains) > 0 { + return fmt.Errorf("some chains in %s are jumped to but have no rules: %v", tableName, emptyChains.List()) + } + + // FIXME: This currently fails + // extraChains := createdChains.Difference(jumpedChains) + // extraChains.Delete(string(kubeServicesChain), string(kubeExternalServicesChain), string(kubeNodePortsChain), string(kubePostroutingChain), string(kubeForwardChain), string(KubeMarkMasqChain)) + // if len(extraChains) > 0 { + // return fmt.Errorf("some chains in %s are created but not used: %v", tableName, extraChains.List()) + // } + + return nil +} + // sortIPTablesRules sorts `iptables-restore` output so as to not depend on the order that // Services get processed in, while preserving the relative ordering of related rules. func sortIPTablesRules(ruleData string) (string, error) { @@ -579,6 +616,11 @@ func sortIPTablesRules(ruleData string) (string, error) { for _, table := range tables { lines := strings.Split(strings.Trim(table, "\n"), "\n") + err := assertIPTablesRuleJumps(lines) + if err != nil { + return "", err + } + // Move "*TABLENAME" line lines, output = moveMatchingLines(`^\*`, lines, output) @@ -946,6 +988,45 @@ COMMIT COMMIT `, }, + { + name: "can't jump to chain that wasn't created", + input: ` +*filter +COMMIT +*nat +:KUBE-SERVICES - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +COMMIT +`, + error: "some chains in *nat are used but were not created: [KUBE-SVC-XPGD46QRK7WJZT7O]", + }, + { + name: "can't jump to chain that has no rules", + input: ` +*filter +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +COMMIT +`, + error: "some chains in *nat are jumped to but have no rules: [KUBE-SVC-XPGD46QRK7WJZT7O]", + }, + { + name: "can't add rules to a chain that wasn't created", + input: ` +*filter +COMMIT +*nat +:KUBE-MARK-MASQ - [0:0] +:KUBE-SERVICES - [0:0] +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" ... +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +COMMIT +`, + error: "some chains in *nat are used but were not created: [KUBE-SVC-XPGD46QRK7WJZT7O]", + }, } { t.Run(tc.name, func(t *testing.T) { out, err := sortIPTablesRules(tc.input) From 22a951c09694f19d53a8b996c5ef7bf926379382 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 29 Oct 2021 12:00:30 -0400 Subject: [PATCH 5/8] proxy/iptables: Fix TestOnlyLocalNodePortsNoClusterCIDR The test got broken to not actually use "no cluster CIDR" when LocalDetector was implemented (and the old version of the unit test didn't check enough to actually notice this). --- pkg/proxy/iptables/proxier_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index d21c5e76423..bf8fe6fbba0 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -2136,6 +2136,7 @@ COMMIT func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) + fp.localDetector = proxyutiliptables.NewNoOpLocalDetector() expected := ` *filter @@ -2161,7 +2162,6 @@ COMMIT -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 --A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -s 127.0.0.0/8 -j KUBE-MARK-MASQ -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-XLB-XPGD46QRK7WJZT7O @@ -2171,7 +2171,6 @@ COMMIT -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -s 10.180.2.1/32 -j KUBE-MARK-MASQ -A KUBE-SEP-ZX7GRIZKSNUQ3LAJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.2.1:80 --A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "masquerade LOCAL traffic for ns1/svc1:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "route LOCAL traffic for ns1/svc1:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-XPGD46QRK7WJZT7O -A KUBE-XLB-XPGD46QRK7WJZT7O -m comment --comment "Balancing rule 0 for ns1/svc1:p80" -j KUBE-SEP-ZX7GRIZKSNUQ3LAJ From 6ab3dc6875e79c8783ae1934dba10f06ceae64b1 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 27 Oct 2021 17:51:11 -0400 Subject: [PATCH 6/8] proxy/iptables: Add more stuff to the unit test The external traffic policy terminating endpoints test was testing LoadBalancer functionality against a NodePort service with no nodePorts (or loadBalancer IPs). It managed to test what it wanted to test, but it's kind of dubious (and we probably _shouldn't_ have been generating the rules it was looking for since there was no way to actually reach the XLB chains). So fix that. Also make the terminating endpoints test use session affinity, to add more testing for that. Also, remove the multiple copies of the same identical Service that is used for all of the test cases in that test. Also add a "Cluster traffic policy and no source ranges" test to TestOverallIPTablesRulesWithMultipleServices since we weren't really testing either of those. Also add a test of --masquerade-all. --- pkg/proxy/iptables/proxier_test.go | 306 ++++++++++++++++++++--------- 1 file changed, 216 insertions(+), 90 deletions(-) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index bf8fe6fbba0..c1ac27ebcaf 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -1087,6 +1087,22 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"} svc.Spec.HealthCheckNodePort = 30000 }), + // create LoadBalancer service with Cluster traffic policy and no source ranges + makeTestService("ns2b", "svc2b", func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster + svc.Spec.ClusterIP = "10.20.30.43" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: 3001, + }} + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: "1.2.3.5", + }} + svc.Spec.HealthCheckNodePort = 30000 + }), // create NodePort service makeTestService("ns3", "svc3", func(svc *v1.Service) { svc.Spec.Type = "NodePort" @@ -1124,7 +1140,8 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { Protocol: &tcpProtocol, }} }), - // create LoadBalancer endpoints + // create Local LoadBalancer endpoints. Note that since we aren't setting + // its NodeName, this endpoint will be considered non-local and ignored. makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ @@ -1136,6 +1153,18 @@ func TestOverallIPTablesRulesWithMultipleServices(t *testing.T) { Protocol: &tcpProtocol, }} }), + // create Cluster LoadBalancer endpoints + makeTestEndpointSlice("ns2b", "svc2b", 1, func(eps *discovery.EndpointSlice) { + eps.AddressType = discovery.AddressTypeIPv4 + eps.Endpoints = []discovery.Endpoint{{ + Addresses: []string{"10.180.0.3"}, + }} + eps.Ports = []discovery.EndpointPort{{ + Name: utilpointer.StringPtr("p80"), + Port: utilpointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), // create NodePort service endpoints makeTestEndpointSlice("ns3", "svc3", 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 @@ -1190,6 +1219,9 @@ COMMIT :KUBE-XLB-GNZBNJ2PO5MGZ6GT - [0:0] :KUBE-FW-GNZBNJ2PO5MGZ6GT - [0:0] :KUBE-SEP-RS4RBKLTHTF2IUXJ - [0:0] +:KUBE-SVC-PAZTZYUUMV5KCDZL - [0:0] +:KUBE-FW-PAZTZYUUMV5KCDZL - [0:0] +:KUBE-SEP-QDCEFMBQEGWIV4VT - [0:0] :KUBE-SVC-X27LE4BHSL4DOUIK - [0:0] :KUBE-SEP-OYPFS5VJICHGATKP - [0:0] :KUBE-SVC-4SW47YFZTEDKD3PK - [0:0] @@ -1219,6 +1251,17 @@ COMMIT -A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "route LOCAL traffic for ns2/svc2:p80 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-GNZBNJ2PO5MGZ6GT -A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "ns2/svc2:p80 has no local endpoints" -j KUBE-MARK-DROP +-A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 -j KUBE-SVC-PAZTZYUUMV5KCDZL +-A KUBE-SERVICES -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.5/32 --dport 80 -j KUBE-FW-PAZTZYUUMV5KCDZL +-A KUBE-FW-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -j KUBE-MARK-MASQ +-A KUBE-FW-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -j KUBE-SVC-PAZTZYUUMV5KCDZL +-A KUBE-FW-PAZTZYUUMV5KCDZL -m comment --comment "ns2b/svc2b:p80 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-PAZTZYUUMV5KCDZL +-A KUBE-SVC-PAZTZYUUMV5KCDZL -m comment --comment ns2b/svc2b:p80 -j KUBE-SEP-QDCEFMBQEGWIV4VT +-A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -s 10.180.0.3/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-QDCEFMBQEGWIV4VT -m comment --comment ns2b/svc2b:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.3:80 -A KUBE-SERVICES -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 -j KUBE-SVC-X27LE4BHSL4DOUIK -A KUBE-SVC-X27LE4BHSL4DOUIK -m comment --comment "ns3/svc3:p80 cluster IP" -m tcp -p tcp -d 10.20.30.43/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ -A KUBE-NODEPORTS -m comment --comment ns3/svc3:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-X27LE4BHSL4DOUIK @@ -4221,33 +4264,47 @@ COMMIT // endpoints, only the ready endpoints are used. func Test_EndpointSliceWithTerminatingEndpoints(t *testing.T) { tcpProtocol := v1.ProtocolTCP + timeout := v1.DefaultClientIPServiceAffinitySeconds + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"}, + Spec: v1.ServiceSpec{ + ClusterIP: "172.20.1.1", + Type: v1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, + Selector: map[string]string{"foo": "bar"}, + Ports: []v1.ServicePort{ + { + Name: "", + TargetPort: intstr.FromInt(80), + Port: 80, + Protocol: v1.ProtocolTCP, + }, + }, + SessionAffinity: v1.ServiceAffinityClientIP, + SessionAffinityConfig: &v1.SessionAffinityConfig{ + ClientIP: &v1.ClientIPConfig{ + TimeoutSeconds: &timeout, + }, + }, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + {IP: "10.1.2.3"}, + }, + }, + }, + } + testcases := []struct { name string terminatingFeatureGate bool - service *v1.Service endpointslice *discovery.EndpointSlice expectedIPTables string }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", terminatingFeatureGate: true, - service: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"}, - Spec: v1.ServiceSpec{ - ClusterIP: "172.20.1.1", - Type: v1.ServiceTypeNodePort, - ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{ - { - Name: "", - TargetPort: intstr.FromInt(80), - Port: 80, - Protocol: v1.ProtocolTCP, - }, - }, - }, - }, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-1", "svc1"), @@ -4329,6 +4386,7 @@ COMMIT :KUBE-MARK-MASQ - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] +:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] @@ -4340,22 +4398,32 @@ COMMIT -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 +-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 +-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 +-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 -A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 +-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 +-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 0 for ns1/svc1" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 1 for ns1/svc1" -j KUBE-SEP-IO5XOSKPAXIFQXAJ -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS @@ -4365,23 +4433,6 @@ COMMIT { name: "feature gate ProxyTerminatingEndpoints disabled, ready endpoints exist", terminatingFeatureGate: false, - service: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"}, - Spec: v1.ServiceSpec{ - ClusterIP: "172.20.1.1", - Type: v1.ServiceTypeNodePort, - ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{ - { - Name: "", - TargetPort: intstr.FromInt(80), - Port: 80, - Protocol: v1.ProtocolTCP, - }, - }, - }, - }, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-1", "svc1"), @@ -4463,6 +4514,7 @@ COMMIT :KUBE-MARK-MASQ - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] +:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] @@ -4474,22 +4526,32 @@ COMMIT -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 +-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 +-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 +-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 -A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 +-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 +-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 0 for ns1/svc1" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 1 for ns1/svc1" -j KUBE-SEP-IO5XOSKPAXIFQXAJ -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS @@ -4499,23 +4561,6 @@ COMMIT { name: "feature gate ProxyTerminatingEndpoints enabled, only terminating endpoints exist", terminatingFeatureGate: true, - service: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"}, - Spec: v1.ServiceSpec{ - ClusterIP: "172.20.1.1", - Type: v1.ServiceTypeNodePort, - ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{ - { - Name: "", - TargetPort: intstr.FromInt(80), - Port: 80, - Protocol: v1.ProtocolTCP, - }, - }, - }, - }, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-1", "svc1"), @@ -4589,6 +4634,7 @@ COMMIT :KUBE-MARK-MASQ - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] +:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] :KUBE-SEP-VLJB2F747S6W7EX4 - [0:0] @@ -4599,18 +4645,27 @@ COMMIT -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 +-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 +-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 -A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 +-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 +-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 0 for ns1/svc1" -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Balancing rule 1 for ns1/svc1" -j KUBE-SEP-XGJFVO3L2O5SRFNT -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS @@ -4620,23 +4675,6 @@ COMMIT { name: "with ProxyTerminatingEndpoints disabled, only terminating endpoints exist", terminatingFeatureGate: false, - service: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "svc1", Namespace: "ns1"}, - Spec: v1.ServiceSpec{ - ClusterIP: "172.20.1.1", - Type: v1.ServiceTypeNodePort, - ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, - Selector: map[string]string{"foo": "bar"}, - Ports: []v1.ServicePort{ - { - Name: "", - TargetPort: intstr.FromInt(80), - Port: 80, - Protocol: v1.ProtocolTCP, - }, - }, - }, - }, endpointslice: &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-1", "svc1"), @@ -4715,6 +4753,7 @@ COMMIT :KUBE-MARK-MASQ - [0:0] :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] +:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] @@ -4726,17 +4765,25 @@ COMMIT -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ -A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 +-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 +-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 +-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 -A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 +-A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ --A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 +-A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4756,7 +4803,7 @@ COMMIT fp.OnServiceSynced() fp.OnEndpointSlicesSynced() - fp.OnServiceAdd(testcase.service) + fp.OnServiceAdd(service) fp.OnEndpointSliceAdd(testcase.endpointslice) fp.syncProxyRules() @@ -4769,3 +4816,82 @@ COMMIT }) } } + +func TestMasqueradeAll(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + fp.masqueradeAll = true + + makeServiceMap(fp, + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = "10.20.30.41" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: 80, + Protocol: v1.ProtocolTCP, + NodePort: int32(3001), + }} + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ + IP: "1.2.3.4", + }} + }), + ) + + tcpProtocol := v1.ProtocolTCP + populateEndpointSlices(fp, + makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { + eps.AddressType = discovery.AddressTypeIPv4 + eps.Endpoints = []discovery.Endpoint{{ + Addresses: []string{"10.180.0.1"}, + }} + eps.Ports = []discovery.EndpointPort{{ + Name: utilpointer.StringPtr("p80"), + Port: utilpointer.Int32(80), + Protocol: &tcpProtocol, + }} + }), + ) + + fp.syncProxyRules() + + expected := ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] +:KUBE-FW-XPGD46QRK7WJZT7O - [0:0] +:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 loadbalancer IP" -m tcp -p tcp -d 1.2.3.4/32 --dport 80 -j KUBE-FW-XPGD46QRK7WJZT7O +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-MASQ +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-FW-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-MARK-MASQ +-A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m tcp -p tcp --dport 3001 -j KUBE-SVC-XPGD46QRK7WJZT7O +-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1/32 -j KUBE-MARK-MASQ +-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +` + + assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) +} From 3679639cf147db8b99994f5f069fbd244ea63a48 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Tue, 2 Nov 2021 17:19:16 -0400 Subject: [PATCH 7/8] proxy/iptables: Remove a no-op check There was code to deal with endpoints that have invalid/empty IP addresses, but EndpointSlice validation already ensures that these can't exist. --- pkg/proxy/iptables/proxier.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index c2b19497f73..24b21375e72 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1387,12 +1387,6 @@ func (proxier *Proxier) syncProxyRules() { // Now write loadbalancing & DNAT rules. numReadyEndpoints := len(readyEndpointChains) for i, endpointChain := range readyEndpointChains { - epIP := readyEndpoints[i].IP() - if epIP == "" { - // Error parsing this endpoint has been logged. Skip to next endpoint. - continue - } - // Balancing rules in the per-service chain. args = append(args[:0], "-A", string(svcChain)) args = proxier.appendServiceCommentLocked(args, svcNameString) @@ -1411,18 +1405,12 @@ func (proxier *Proxier) syncProxyRules() { // Every endpoint gets a chain, regardless of its state. This is required later since we may // want to jump to endpoint chains that are terminating. for i, endpointChain := range endpointChains { - epIP := endpoints[i].IP() - if epIP == "" { - // Error parsing this endpoint has been logged. Skip to next endpoint. - continue - } - // Rules in the per-endpoint chain. args = append(args[:0], "-A", string(endpointChain)) args = proxier.appendServiceCommentLocked(args, svcNameString) // Handle traffic that loops back to the originator with SNAT. utilproxy.WriteLine(proxier.natRules, append(args, - "-s", utilproxy.ToCIDR(netutils.ParseIPSloppy(epIP)), + "-s", utilproxy.ToCIDR(netutils.ParseIPSloppy(endpoints[i].IP())), "-j", string(KubeMarkMasqChain))...) // Update client-affinity lists. if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { From 229ae585209237475a0bd68f1dfab26df0f9024c Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Tue, 2 Nov 2021 11:34:38 -0400 Subject: [PATCH 8/8] proxy/iptables: fix all-vs-ready endpoints a bit Filter the allEndpoints list into readyEndpoints sooner, and set "hasEndpoints" based (mostly) on readyEndpoints, not allEndpoints (so that, eg, we correctly generate REJECT rules for services with no _functioning_ endpoints, even if they have unusable terminating endpoints). Also, write out the endpoint chains at the top of the loop when we iterate the endpoints for the first time, rather than copying some of the data to another set of variables and then writing them out later. And don't write out endpoint chains that won't be used Also, generate affinity rules only for readyEndpoints rather than allEndpoints, so affinity gets broken correctly when an endpoint becomes unready. --- pkg/proxy/iptables/proxier.go | 161 +++++++++++++------------ pkg/proxy/iptables/proxier_test.go | 183 ++++++++++++++++++++++------- 2 files changed, 217 insertions(+), 127 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 24b21375e72..15323340c8e 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -955,12 +955,8 @@ func (proxier *Proxier) syncProxyRules() { // you should always do one of the below: // slice = slice[:0] // and then append to it // slice = append(slice[:0], ...) - endpoints := make([]*endpointsInfo, 0) - endpointChains := make([]utiliptables.Chain, 0) - readyEndpoints := make([]*endpointsInfo, 0) readyEndpointChains := make([]utiliptables.Chain, 0) - localReadyEndpointChains := make([]utiliptables.Chain, 0) - localServingTerminatingEndpointChains := make([]utiliptables.Chain, 0) + localEndpointChains := make([]utiliptables.Chain, 0) // To avoid growing this slice, we arbitrarily set its size to 64, // there is never more than that many arguments for a single line. @@ -1002,7 +998,82 @@ func (proxier *Proxier) syncProxyRules() { // Service does not have conflicting configuration such as // externalTrafficPolicy=Local. allEndpoints = proxy.FilterEndpoints(allEndpoints, svcInfo, proxier.nodeLabels) - hasEndpoints := len(allEndpoints) > 0 + + // Scan the endpoints list to see what we have. "hasEndpoints" will be true + // if there are any usable endpoints for this service anywhere in the cluster. + var hasEndpoints, hasLocalReadyEndpoints, hasLocalServingTerminatingEndpoints bool + for _, ep := range allEndpoints { + if ep.IsReady() { + hasEndpoints = true + if ep.GetIsLocal() { + hasLocalReadyEndpoints = true + } + } else if svc.NodeLocalExternal() && utilfeature.DefaultFeatureGate.Enabled(features.ProxyTerminatingEndpoints) { + if ep.IsServing() && ep.IsTerminating() { + hasEndpoints = true + if ep.GetIsLocal() { + hasLocalServingTerminatingEndpoints = true + } + } + } + } + useTerminatingEndpoints := !hasLocalReadyEndpoints && hasLocalServingTerminatingEndpoints + + // Generate the per-endpoint chains. + readyEndpointChains = readyEndpointChains[:0] + localEndpointChains = localEndpointChains[:0] + for _, ep := range allEndpoints { + epInfo, ok := ep.(*endpointsInfo) + if !ok { + klog.ErrorS(err, "Failed to cast endpointsInfo", "endpointsInfo", ep) + continue + } + + endpointChain := epInfo.endpointChain(svcNameString, protocol) + endpointInUse := false + + if epInfo.Ready { + readyEndpointChains = append(readyEndpointChains, endpointChain) + endpointInUse = true + } + if svc.NodeLocalExternal() && epInfo.IsLocal { + if useTerminatingEndpoints { + if epInfo.Serving && epInfo.Terminating { + localEndpointChains = append(localEndpointChains, endpointChain) + endpointInUse = true + } + } else if epInfo.Ready { + localEndpointChains = append(localEndpointChains, endpointChain) + endpointInUse = true + } + } + + if !endpointInUse { + continue + } + + // Create the endpoint chain, retaining counters if possible. + if chain, ok := existingNATChains[endpointChain]; ok { + utilproxy.WriteBytesLine(proxier.natChains, chain) + } else { + utilproxy.WriteLine(proxier.natChains, utiliptables.MakeChainLine(endpointChain)) + } + activeNATChains[endpointChain] = true + + args = append(args[:0], "-A", string(endpointChain)) + args = proxier.appendServiceCommentLocked(args, svcNameString) + // Handle traffic that loops back to the originator with SNAT. + utilproxy.WriteLine(proxier.natRules, append(args, + "-s", utilproxy.ToCIDR(netutils.ParseIPSloppy(epInfo.IP())), + "-j", string(KubeMarkMasqChain))...) + // Update client-affinity lists. + if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { + args = append(args, "-m", "recent", "--name", string(endpointChain), "--set") + } + // DNAT to final destination. + args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", epInfo.Endpoint) + utilproxy.WriteLine(proxier.natRules, args...) + } svcChain := svcInfo.servicePortChainName if hasEndpoints { @@ -1319,35 +1390,9 @@ func (proxier *Proxier) syncProxyRules() { continue } - // Generate the per-endpoint chains. We do this in multiple passes so we - // can group rules together. - // These two slices parallel each other - keep in sync - endpoints = endpoints[:0] - endpointChains = endpointChains[:0] - var endpointChain utiliptables.Chain - for _, ep := range allEndpoints { - epInfo, ok := ep.(*endpointsInfo) - if !ok { - klog.ErrorS(err, "Failed to cast endpointsInfo", "endpointsInfo", ep) - continue - } - - endpoints = append(endpoints, epInfo) - endpointChain = epInfo.endpointChain(svcNameString, protocol) - endpointChains = append(endpointChains, endpointChain) - - // Create the endpoint chain, retaining counters if possible. - if chain, ok := existingNATChains[endpointChain]; ok { - utilproxy.WriteBytesLine(proxier.natChains, chain) - } else { - utilproxy.WriteLine(proxier.natChains, utiliptables.MakeChainLine(endpointChain)) - } - activeNATChains[endpointChain] = true - } - // First write session affinity rules, if applicable. if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - for _, endpointChain := range endpointChains { + for _, endpointChain := range readyEndpointChains { args = append(args[:0], "-A", string(svcChain), ) @@ -1361,30 +1406,7 @@ func (proxier *Proxier) syncProxyRules() { } } - // Firstly, categorize each endpoint into three buckets: - // 1. all endpoints that are ready and NOT terminating. - // 2. all endpoints that are local, ready and NOT terminating, and externalTrafficPolicy=Local - // 3. all endpoints that are local, serving and terminating, and externalTrafficPolicy=Local - readyEndpointChains = readyEndpointChains[:0] - readyEndpoints := readyEndpoints[:0] - localReadyEndpointChains := localReadyEndpointChains[:0] - localServingTerminatingEndpointChains := localServingTerminatingEndpointChains[:0] - for i, endpointChain := range endpointChains { - if endpoints[i].Ready { - readyEndpointChains = append(readyEndpointChains, endpointChain) - readyEndpoints = append(readyEndpoints, endpoints[i]) - } - - if svc.NodeLocalExternal() && endpoints[i].IsLocal { - if endpoints[i].Ready { - localReadyEndpointChains = append(localReadyEndpointChains, endpointChain) - } else if endpoints[i].Serving && endpoints[i].Terminating { - localServingTerminatingEndpointChains = append(localServingTerminatingEndpointChains, endpointChain) - } - } - } - - // Now write loadbalancing & DNAT rules. + // Now write loadbalancing rules numReadyEndpoints := len(readyEndpointChains) for i, endpointChain := range readyEndpointChains { // Balancing rules in the per-service chain. @@ -1402,25 +1424,6 @@ func (proxier *Proxier) syncProxyRules() { utilproxy.WriteLine(proxier.natRules, args...) } - // Every endpoint gets a chain, regardless of its state. This is required later since we may - // want to jump to endpoint chains that are terminating. - for i, endpointChain := range endpointChains { - // Rules in the per-endpoint chain. - args = append(args[:0], "-A", string(endpointChain)) - args = proxier.appendServiceCommentLocked(args, svcNameString) - // Handle traffic that loops back to the originator with SNAT. - utilproxy.WriteLine(proxier.natRules, append(args, - "-s", utilproxy.ToCIDR(netutils.ParseIPSloppy(endpoints[i].IP())), - "-j", string(KubeMarkMasqChain))...) - // Update client-affinity lists. - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - args = append(args, "-m", "recent", "--name", string(endpointChain), "--set") - } - // DNAT to final destination. - args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", endpoints[i].Endpoint) - utilproxy.WriteLine(proxier.natRules, args...) - } - // The logic below this applies only if this service is marked as OnlyLocal if !svcInfo.NodeLocalExternal() { continue @@ -1449,12 +1452,6 @@ func (proxier *Proxier) syncProxyRules() { "-m", "comment", "--comment", fmt.Sprintf(`"route LOCAL traffic for %s LB IP to service chain"`, svcNameString), "-m", "addrtype", "--src-type", "LOCAL", "-j", string(svcChain))...) - // Prefer local ready endpoint chains, but fall back to ready terminating if none exist - localEndpointChains := localReadyEndpointChains - if utilfeature.DefaultFeatureGate.Enabled(features.ProxyTerminatingEndpoints) && len(localEndpointChains) == 0 { - localEndpointChains = localServingTerminatingEndpointChains - } - numLocalEndpoints := len(localEndpointChains) if numLocalEndpoints == 0 { // Blackhole all traffic since there are no local endpoints diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index c1ac27ebcaf..66eb9bcf92d 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -3532,7 +3532,6 @@ COMMIT :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] -:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0] -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE @@ -3548,8 +3547,6 @@ COMMIT -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS COMMIT ` @@ -3635,7 +3632,6 @@ COMMIT :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] -:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0] -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE @@ -3653,8 +3649,6 @@ COMMIT -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4301,6 +4295,7 @@ func Test_EndpointSliceWithTerminatingEndpoints(t *testing.T) { terminatingFeatureGate bool endpointslice *discovery.EndpointSlice expectedIPTables string + noUsableEndpoints bool }{ { name: "feature gate ProxyTerminatingEndpoints enabled, ready endpoints exist", @@ -4389,8 +4384,6 @@ COMMIT :KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] -:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0] :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4403,8 +4396,6 @@ COMMIT -A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ @@ -4413,10 +4404,6 @@ COMMIT -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4517,8 +4504,6 @@ COMMIT :KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] -:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0] :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4531,8 +4516,6 @@ COMMIT -A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ @@ -4541,10 +4524,6 @@ COMMIT -A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4637,7 +4616,6 @@ COMMIT :KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] :KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] :KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] -:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0] :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4648,17 +4626,12 @@ COMMIT -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP -A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP -A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ -A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ -A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4754,10 +4727,6 @@ COMMIT :KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] :KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] :KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] -:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0] -:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0] -:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0] -:KUBE-SEP-VLJB2F747S6W7EX4 - [0:0] :KUBE-SEP-EQCHZ7S2PJ72OHAY - [0:0] -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 @@ -4768,20 +4737,8 @@ COMMIT -A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP -A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP -A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-3JOIVZTXZZRGORX4 --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --rcheck --seconds 10800 --reap -j KUBE-SEP-IO5XOSKPAXIFQXAJ --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --rcheck --seconds 10800 --reap -j KUBE-SEP-XGJFVO3L2O5SRFNT --A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --rcheck --seconds 10800 --reap -j KUBE-SEP-VLJB2F747S6W7EX4 -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --rcheck --seconds 10800 --reap -j KUBE-SEP-EQCHZ7S2PJ72OHAY -A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-EQCHZ7S2PJ72OHAY --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ --A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-3JOIVZTXZZRGORX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80 --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ --A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-IO5XOSKPAXIFQXAJ --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80 --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ --A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-XGJFVO3L2O5SRFNT --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80 --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -s 10.0.1.4/32 -j KUBE-MARK-MASQ --A KUBE-SEP-VLJB2F747S6W7EX4 -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-VLJB2F747S6W7EX4 --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.4:80 -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -s 10.0.1.5/32 -j KUBE-MARK-MASQ -A KUBE-SEP-EQCHZ7S2PJ72OHAY -m comment --comment ns1/svc1 -m recent --name KUBE-SEP-EQCHZ7S2PJ72OHAY --set -m tcp -p tcp -j DNAT --to-destination 10.0.1.5:80 -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP @@ -4790,6 +4747,137 @@ COMMIT -A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS COMMIT +`, + }, + { + name: "ProxyTerminatingEndpoints enabled, terminating endpoints on remote node", + terminatingFeatureGate: true, + endpointslice: &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-1", "svc1"), + Namespace: "ns1", + Labels: map[string]string{discovery.LabelServiceName: "svc1"}, + }, + Ports: []discovery.EndpointPort{{ + Name: utilpointer.StringPtr(""), + Port: utilpointer.Int32Ptr(80), + Protocol: &tcpProtocol, + }}, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{ + { + // this endpoint won't be used because it's not local, + // but it will prevent a REJECT rule from being created + Addresses: []string{"10.0.1.5"}, + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(false), + Serving: utilpointer.BoolPtr(true), + Terminating: utilpointer.BoolPtr(true), + }, + NodeName: utilpointer.StringPtr("host-1"), + }, + }, + }, + expectedIPTables: ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0] +:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] +:KUBE-FW-AQI2S6QIMU7PVVRP - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-SERVICES -m comment --comment "ns1/svc1 loadbalancer IP" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j KUBE-FW-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-XLB-AQI2S6QIMU7PVVRP +-A KUBE-FW-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 loadbalancer IP" -j KUBE-MARK-DROP +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "Redirect pods trying to reach external loadbalancer VIP to clusterIP" -s 10.0.0.0/24 -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "masquerade LOCAL traffic for ns1/svc1 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "route LOCAL traffic for ns1/svc1 LB IP to service chain" -m addrtype --src-type LOCAL -j KUBE-SVC-AQI2S6QIMU7PVVRP +-A KUBE-XLB-AQI2S6QIMU7PVVRP -m comment --comment "ns1/svc1 has no local endpoints" -j KUBE-MARK-DROP +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT +`, + }, + { + name: "no usable endpoints on any node", + terminatingFeatureGate: true, + endpointslice: &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-1", "svc1"), + Namespace: "ns1", + Labels: map[string]string{discovery.LabelServiceName: "svc1"}, + }, + Ports: []discovery.EndpointPort{{ + Name: utilpointer.StringPtr(""), + Port: utilpointer.Int32Ptr(80), + Protocol: &tcpProtocol, + }}, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{ + { + // Local but not ready or serving + Addresses: []string{"10.0.1.5"}, + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(false), + Serving: utilpointer.BoolPtr(false), + Terminating: utilpointer.BoolPtr(true), + }, + NodeName: utilpointer.StringPtr(testHostname), + }, + { + // Remote and not ready or serving + Addresses: []string{"10.0.1.5"}, + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(false), + Serving: utilpointer.BoolPtr(false), + Terminating: utilpointer.BoolPtr(true), + }, + NodeName: utilpointer.StringPtr("host-1"), + }, + }, + }, + noUsableEndpoints: true, + expectedIPTables: ` +*filter +:KUBE-SERVICES - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-NODEPORTS - [0:0] +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j REJECT +-A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 10.1.2.3/32 --dport 80 -j REJECT +COMMIT +*nat +:KUBE-SERVICES - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-XLB-AQI2S6QIMU7PVVRP - [0:0] +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE +-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +COMMIT `, }, } @@ -4812,7 +4900,12 @@ COMMIT fp.OnEndpointSliceDelete(testcase.endpointslice) fp.syncProxyRules() - assertIPTablesRulesNotEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + if testcase.noUsableEndpoints { + // Deleting the EndpointSlice should have had no effect + assertIPTablesRulesEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + } else { + assertIPTablesRulesNotEqual(t, testcase.expectedIPTables, fp.iptablesData.String()) + } }) } }