github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 03:11:40 +00:00
Code Issues Projects Releases Wiki Activity

kube-proxy iptables expose number of rules metrics

Browse Source 
add a new metric to kube-proxy iptables, so it exposes the number
of rules programmed in each iteration.
This commit is contained in:
Antonio Ojea 2021-03-02 18:35:32 +01:00
parent f79795d718
commit 654be57022
5 changed files with 203 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/proxy/iptables/proxier.go
View File

@ -1604,6 +1604,11 @@ func (proxier *Proxier) syncProxyRules() {
proxier.iptablesData.Write(proxier.natChains.Bytes())
proxier.iptablesData.Write(proxier.natRules.Bytes())
numberFilterIptablesRules := utilproxy.CountBytesLines(proxier.filterRules.Bytes())
metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)).Set(float64(numberFilterIptablesRules))
numberNatIptablesRules := utilproxy.CountBytesLines(proxier.natRules.Bytes())
metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)).Set(float64(numberNatIptablesRules))
klog.V(5).InfoS("Restoring iptables", "rules", proxier.iptablesData.Bytes())
err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters)
if err != nil {

123
pkg/proxy/iptables/proxier_test.go
View File

@ -32,8 +32,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/component-base/metrics/testutil"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/metrics"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
utilproxy "k8s.io/kubernetes/pkg/proxy/util"
proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
@ -2932,4 +2935,124 @@ func TestProxierDeleteNodePortStaleUDP(t *testing.T) {
}
}
func TestProxierMetricsIptablesTotalRules(t *testing.T) {
ipt := iptablestest.NewFake()
fp := NewFakeProxier(ipt, false)
metrics.RegisterMetrics()
svcIP := "10.20.30.41"
svcPort := 80
nodePort := 31201
svcPortName := proxy.ServicePortName{
NamespacedName: makeNSN("ns1", "svc1"),
Port: "p80",
Protocol: v1.ProtocolTCP,
}
makeServiceMap(fp,
makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
svc.Spec.ClusterIP = svcIP
svc.Spec.Ports = []v1.ServicePort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
NodePort: int32(nodePort),
}}
}),
)
makeEndpointsMap(fp)
fp.syncProxyRules()
nFilterRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)))
if err != nil {
t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err)
}
// -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m udp -p udp -d 10.20.30.41/32 --dport 80 -j REJECT
// -A KUBE-EXTERNAL-SERVICES -m comment --comment "ns1/svc1:p80 has no endpoints" -m addrtype --dst-type LOCAL -m udp -p udp --dport 31201 -j REJECT
// -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
// -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
// -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
// -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
// COMMIT
if nFilterRules != 7.0 {
t.Fatalf("Wrong number of filter rule: expected 7 received %f", nFilterRules)
}
nNatRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)))
if err != nil {
t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err)
}
// rules -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
// -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
// -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
// -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
// -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
// COMMIT
if nNatRules != 6.0 {
t.Fatalf("Wrong number of nat rules: expected 6 received %f", nNatRules)
}
makeEndpointsMap(fp,
makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
ept.Subsets = []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{
{
IP: "10.0.0.2",
},
{
IP: "10.0.0.5",
},
},
Ports: []v1.EndpointPort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
}},
}}
}),
)
fp.syncProxyRules()
nFilterRules, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)))
if err != nil {
t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err)
}
// -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
// -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
// -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
// -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
// COMMIT
if nFilterRules != 5.0 {
t.Fatalf("Wrong number of filter rule: expected 5 received %f", nFilterRules)
}
nNatRules, err = testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)))
if err != nil {
t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err)
}
// -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
// -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
// -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
// -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
// -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m udp -p udp -d 10.20.30.41/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ
// -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m udp -p udp -d 10.20.30.41/32 --dport 80 -j KUBE-SVC-OJWW7NSBVZTDHXNW
// -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m udp -p udp --dport 31201 -j KUBE-MARK-MASQ
// -A KUBE-NODEPORTS -m comment --comment ns1/svc1:p80 -m udp -p udp --dport 31201 -j KUBE-SVC-OJWW7NSBVZTDHXNW
// -A KUBE-SVC-OJWW7NSBVZTDHXNW -m comment --comment ns1/svc1:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-AMT2SNW3YUNHJFJG
// -A KUBE-SEP-AMT2SNW3YUNHJFJG -m comment --comment ns1/svc1:p80 -s 10.0.0.2/32 -j KUBE-MARK-MASQ
// -A KUBE-SEP-AMT2SNW3YUNHJFJG -m comment --comment ns1/svc1:p80 -m udp -p udp -j DNAT --to-destination 10.0.0.2:80
// -A KUBE-SVC-OJWW7NSBVZTDHXNW -m comment --comment ns1/svc1:p80 -j KUBE-SEP-OUFLBLJVR33W4FIZ
// -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -s 10.0.0.5/32 -j KUBE-MARK-MASQ
// -A KUBE-SEP-OUFLBLJVR33W4FIZ -m comment --comment ns1/svc1:p80 -m udp -p udp -j DNAT --to-destination 10.0.0.5:80
// -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
// COMMIT
if nNatRules != 16.0 {
t.Fatalf("Wrong number of nat rules: expected 16 received %f", nNatRules)
}
}
// TODO(thockin): add *more* tests for syncProxyRules() or break it down further and test the pieces.

12
pkg/proxy/metrics/metrics.go
View File

@ -126,6 +126,17 @@ var (
},
)
// IptablesRulesTotal is the number of iptables rules that the iptables proxy installs.
IptablesRulesTotal = metrics.NewGaugeVec(
&metrics.GaugeOpts{
Subsystem: kubeProxySubsystem,
Name: "sync_proxy_rules_iptables_total",
Help: "Number of proxy iptables rules programmed",
StabilityLevel: metrics.ALPHA,
},
[]string{"table"},
)
// SyncProxyRulesLastQueuedTimestamp is the last time a proxy sync was
// requested. If this is much larger than
// kubeproxy_sync_proxy_rules_last_timestamp_seconds, then something is hung.
@ -151,6 +162,7 @@ func RegisterMetrics() {
legacyregistry.MustRegister(EndpointChangesTotal)
legacyregistry.MustRegister(ServiceChangesPending)
legacyregistry.MustRegister(ServiceChangesTotal)
legacyregistry.MustRegister(IptablesRulesTotal)
legacyregistry.MustRegister(IptablesRestoreFailuresTotal)
legacyregistry.MustRegister(SyncProxyRulesLastQueuedTimestamp)
})

5
pkg/proxy/util/utils.go
View File

@ -494,3 +494,8 @@ func RevertPorts(replacementPortsMap, originalPortsMap map[utilnet.LocalPort]uti
}
}
}
// CountBytesLines counts the number of lines in a bytes slice
func CountBytesLines(b []byte) int {
return bytes.Count(b, []byte{'\n'})
}

58
pkg/proxy/util/utils_test.go
View File

@ -20,6 +20,7 @@ import (
"bytes"
"context"
"fmt"
"math/rand"
"net"
"reflect"
"strings"
@ -1211,3 +1212,60 @@ func TestWriteBytesLine(t *testing.T) {
})
}
}
func TestWriteCountLines(t *testing.T) {
testCases := []struct {
name string
expected int
}{
{
name: "write no line",
expected: 0,
},
{
name: "write one line",
expected: 1,
},
{
name: "write 100 lines",
expected: 100,
},
{
name: "write 1000 lines",
expected: 1000,
},
{
name: "write 10000 lines",
expected: 10000,
},
{
name: "write 100000 lines",
expected: 100000,
},
}
testBuffer := bytes.NewBuffer(nil)
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
testBuffer.Reset()
for i := 0; i < testCase.expected; i++ {
WriteLine(testBuffer, randSeq())
}
n := CountBytesLines(testBuffer.Bytes())
if n != testCase.expected {
t.Fatalf("lines expected: %d, got: %d", testCase.expected, n)
}
})
}
}
// obtained from https://stackoverflow.com/a/22892986
var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
func randSeq() string {
b := make([]rune, 30)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}