diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 33517cb0d96..7464b837da7 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1604,6 +1604,11 @@ func (proxier *Proxier) syncProxyRules() { proxier.iptablesData.Write(proxier.natChains.Bytes()) proxier.iptablesData.Write(proxier.natRules.Bytes()) + numberFilterIptablesRules := utilproxy.CountBytesLines(proxier.filterRules.Bytes()) + metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)).Set(float64(numberFilterIptablesRules)) + numberNatIptablesRules := utilproxy.CountBytesLines(proxier.natRules.Bytes()) + metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)).Set(float64(numberNatIptablesRules)) + klog.V(5).InfoS("Restoring iptables", "rules", proxier.iptablesData.Bytes()) err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 2d4f2d992ec..6d1b8f1bf28 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -32,8 +32,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/component-base/metrics/testutil" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/proxy" + "k8s.io/kubernetes/pkg/proxy/metrics" + "k8s.io/kubernetes/pkg/proxy/healthcheck" utilproxy "k8s.io/kubernetes/pkg/proxy/util" proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" @@ -2932,4 +2935,124 @@ func TestProxierDeleteNodePortStaleUDP(t *testing.T) { } } +func TestProxierMetricsIptablesTotalRules(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt, false) + + metrics.RegisterMetrics() + + svcIP := "10.20.30.41" + svcPort := 80 + nodePort := 31201 + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + Protocol: v1.ProtocolTCP, + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + NodePort: int32(nodePort), + }} + }), + ) + makeEndpointsMap(fp) + + fp.syncProxyRules() + + nFilterRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter))) + if err != nil { + t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) + } + // -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m udp -p udp -d 10.20.30.41/32 --dport 80 -j REJECT + // -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m udp -p udp --dport 31201 -j REJECT + // -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + // COMMIT + + if nFilterRules != 7.0 { + t.Fatalf("Wrong number of filter rule: expected 7 received %f", nFilterRules) + } + + nNatRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) + if err != nil { + t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) + } + + // rules -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + // -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + // -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + // -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + // -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + // COMMIT + if nNatRules != 6.0 { + t.Fatalf("Wrong number of nat rules: expected 6 received %f", nNatRules) + } + + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{ + { + IP: "10.0.0.2", + }, + { + IP: "10.0.0.5", + }, + }, + Ports: []v1.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + }}, + }} + }), + ) + + fp.syncProxyRules() + + nFilterRules, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter))) + if err != nil { + t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) + } + // -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + // -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + // COMMIT + if nFilterRules != 5.0 { + t.Fatalf("Wrong number of filter rule: expected 5 received %f", nFilterRules) + } + nNatRules, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT))) + if err != nil { + t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err) + } + // -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + // -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + // -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + // -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + // -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m udp -p udp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ + // -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m udp -p udp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-OJWW7NSBVZTDHXNW + // -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m udp -p udp --dport 31201 -j KUBE-MARK-MASQ + // -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m udp -p udp --dport 31201 -j KUBE-SVC-OJWW7NSBVZTDHXNW + // -A KUBE-SVC-OJWW7NSBVZTDHXNW -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-AMT2SNW3YUNHJFJG + // -A KUBE-SEP-AMT2SNW3YUNHJFJG -m comment --comment ns1/svc1:p80 -s 10.0.0.2/32 -j KUBE-MARK-MASQ + // -A KUBE-SEP-AMT2SNW3YUNHJFJG -m comment --comment ns1/svc1:p80 -m udp -p udp -j DNAT --to-destination 10.0.0.2:80 + // -A KUBE-SVC-OJWW7NSBVZTDHXNW -m comment --comment ns1/svc1:p80 -j KUBE-SEP-OUFLBLJVR33W4FIZ + // -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -s 10.0.0.5/32 -j KUBE-MARK-MASQ + // -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -m udp -p udp -j DNAT --to-destination 10.0.0.5:80 + // -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + // COMMIT + if nNatRules != 16.0 { + t.Fatalf("Wrong number of nat rules: expected 16 received %f", nNatRules) + } +} + // TODO(thockin): add *more* tests for syncProxyRules() or break it down further and test the pieces. diff --git a/pkg/proxy/metrics/metrics.go b/pkg/proxy/metrics/metrics.go index e917704c7c5..9c1aeca55ba 100644 --- a/pkg/proxy/metrics/metrics.go +++ b/pkg/proxy/metrics/metrics.go @@ -126,6 +126,17 @@ var ( }, ) + // IptablesRulesTotal is the number of iptables rules that the iptables proxy installs. + IptablesRulesTotal = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_iptables_total", + Help: "Number of proxy iptables rules programmed", + StabilityLevel: metrics.ALPHA, + }, + []string{"table"}, + ) + // SyncProxyRulesLastQueuedTimestamp is the last time a proxy sync was // requested. If this is much larger than // kubeproxy_sync_proxy_rules_last_timestamp_seconds, then something is hung. @@ -151,6 +162,7 @@ func RegisterMetrics() { legacyregistry.MustRegister(EndpointChangesTotal) legacyregistry.MustRegister(ServiceChangesPending) legacyregistry.MustRegister(ServiceChangesTotal) + legacyregistry.MustRegister(IptablesRulesTotal) legacyregistry.MustRegister(IptablesRestoreFailuresTotal) legacyregistry.MustRegister(SyncProxyRulesLastQueuedTimestamp) }) diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go index 45b46582de5..9ec44c2de5a 100644 --- a/pkg/proxy/util/utils.go +++ b/pkg/proxy/util/utils.go @@ -494,3 +494,8 @@ func RevertPorts(replacementPortsMap, originalPortsMap map[utilnet.LocalPort]uti } } } + +// CountBytesLines counts the number of lines in a bytes slice +func CountBytesLines(b []byte) int { + return bytes.Count(b, []byte{'\n'}) +} diff --git a/pkg/proxy/util/utils_test.go b/pkg/proxy/util/utils_test.go index f2467405f25..b63463f64a5 100644 --- a/pkg/proxy/util/utils_test.go +++ b/pkg/proxy/util/utils_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "math/rand" "net" "reflect" "strings" @@ -1211,3 +1212,60 @@ func TestWriteBytesLine(t *testing.T) { }) } } + +func TestWriteCountLines(t *testing.T) { + + testCases := []struct { + name string + expected int + }{ + { + name: "write no line", + expected: 0, + }, + { + name: "write one line", + expected: 1, + }, + { + name: "write 100 lines", + expected: 100, + }, + { + name: "write 1000 lines", + expected: 1000, + }, + { + name: "write 10000 lines", + expected: 10000, + }, + { + name: "write 100000 lines", + expected: 100000, + }, + } + testBuffer := bytes.NewBuffer(nil) + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + testBuffer.Reset() + for i := 0; i < testCase.expected; i++ { + WriteLine(testBuffer, randSeq()) + } + n := CountBytesLines(testBuffer.Bytes()) + if n != testCase.expected { + t.Fatalf("lines expected: %d, got: %d", testCase.expected, n) + } + }) + } +} + +// obtained from https://stackoverflow.com/a/22892986 +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func randSeq() string { + b := make([]rune, 30) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +}