mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-17 15:50:10 +00:00
Split IptablesRulesTotal metric into two different metrics
Historically, IptablesRulesTotal could have been intepreted as either "the total number of iptables rules kube-proxy is responsible for" or "the number of iptables rules kube-proxy rewrote on the last sync". Post-MinimizeIPTablesRestore, these are very different things (and IptablesRulesTotal unintentionally became the latter). Fix IptablesRulesTotal (sync_proxy_rules_iptables_total) to be "the total number of iptables rules kube-proxy is responsible for" and add IptablesRulesLastSync (sync_proxy_rules_iptables_last) to be "the number of iptables rules kube-proxy rewrote on the last sync".
This commit is contained in:
Dan Winship
parent
02c59710ea
commit
68ed020b2a
@ -852,6 +852,9 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
proxier.natChains.Reset()
|
||||
proxier.natRules.Reset()
|
||||
|
||||
skippedNatChains := &proxyutil.LineBuffer{}
|
||||
skippedNatRules := &proxyutil.LineBuffer{}
|
||||
|
||||
// Write chain lines for all the "top-level" chains we'll be filling in
|
||||
for _, chainName := range []utiliptables.Chain{kubeServicesChain, kubeExternalServicesChain, kubeForwardChain, kubeNodePortsChain, kubeProxyFirewallChain} {
|
||||
proxier.filterChains.Write(utiliptables.MakeChainLine(chainName))
|
||||
@ -1066,9 +1069,13 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
}
|
||||
}
|
||||
|
||||
filterRules := &proxier.filterRules
|
||||
natChains := &proxier.natChains
|
||||
natRules := &proxier.natRules
|
||||
|
||||
// Capture the clusterIP.
|
||||
if hasInternalEndpoints {
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(kubeServicesChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcPortNameString),
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1077,7 +1084,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
"-j", string(internalTrafficChain))
|
||||
} else {
|
||||
// No endpoints.
|
||||
proxier.filterRules.Write(
|
||||
filterRules.Write(
|
||||
"-A", string(kubeServicesChain),
|
||||
"-m", "comment", "--comment", internalTrafficFilterComment,
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1092,7 +1099,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
if hasEndpoints {
|
||||
// Send traffic bound for external IPs to the "external
|
||||
// destinations" chain.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(kubeServicesChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"%s external IP"`, svcPortNameString),
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1104,7 +1111,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// Either no endpoints at all (REJECT) or no endpoints for
|
||||
// external traffic (DROP anything that didn't get
|
||||
// short-circuited by the EXT chain.)
|
||||
proxier.filterRules.Write(
|
||||
filterRules.Write(
|
||||
"-A", string(kubeExternalServicesChain),
|
||||
"-m", "comment", "--comment", externalTrafficFilterComment,
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1118,7 +1125,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// Capture load-balancer ingress.
|
||||
for _, lbip := range svcInfo.LoadBalancerIPStrings() {
|
||||
if hasEndpoints {
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(kubeServicesChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcPortNameString),
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1128,7 +1135,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
|
||||
}
|
||||
if usesFWChain {
|
||||
proxier.filterRules.Write(
|
||||
filterRules.Write(
|
||||
"-A", string(kubeProxyFirewallChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"%s traffic not accepted by %s"`, svcPortNameString, svcInfo.firewallChainName),
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1142,7 +1149,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// external traffic (DROP anything that didn't get short-circuited
|
||||
// by the EXT chain.)
|
||||
for _, lbip := range svcInfo.LoadBalancerIPStrings() {
|
||||
proxier.filterRules.Write(
|
||||
filterRules.Write(
|
||||
"-A", string(kubeExternalServicesChain),
|
||||
"-m", "comment", "--comment", externalTrafficFilterComment,
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1159,7 +1166,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// Jump to the external destination chain. For better or for
|
||||
// worse, nodeports are not subect to loadBalancerSourceRanges,
|
||||
// and we can't change that.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(kubeNodePortsChain),
|
||||
"-m", "comment", "--comment", svcPortNameString,
|
||||
"-m", protocol, "-p", protocol,
|
||||
@ -1170,7 +1177,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// Either no endpoints at all (REJECT) or no endpoints for
|
||||
// external traffic (DROP anything that didn't get
|
||||
// short-circuited by the EXT chain.)
|
||||
proxier.filterRules.Write(
|
||||
filterRules.Write(
|
||||
"-A", string(kubeExternalServicesChain),
|
||||
"-m", "comment", "--comment", externalTrafficFilterComment,
|
||||
"-m", "addrtype", "--dst-type", "LOCAL",
|
||||
@ -1185,7 +1192,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
if svcInfo.HealthCheckNodePort() != 0 {
|
||||
// no matter if node has local endpoints, healthCheckNodePorts
|
||||
// need to add a rule to accept the incoming connection
|
||||
proxier.filterRules.Write(
|
||||
filterRules.Write(
|
||||
"-A", string(kubeNodePortsChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"%s health check node port"`, svcPortNameString),
|
||||
"-m", "tcp", "-p", "tcp",
|
||||
@ -1196,9 +1203,12 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
|
||||
// If the SVC/SVL/EXT/FW/SEP chains have not changed since the last sync
|
||||
// then we can omit them from the restore input. (We have already marked
|
||||
// them in activeNATChains, so they won't get deleted.)
|
||||
// them in activeNATChains, so they won't get deleted.) However, we have
|
||||
// to still figure out how many chains we _would_ have written to make the
|
||||
// metrics come out right, so we just compute them and throw them away.
|
||||
if tryPartialSync && !serviceChanged.Has(svcName.NamespacedName.String()) && !endpointsChanged.Has(svcName.NamespacedName.String()) {
|
||||
continue
|
||||
natChains = skippedNatChains
|
||||
natRules = skippedNatRules
|
||||
}
|
||||
|
||||
// Set up internal traffic handling.
|
||||
@ -1210,7 +1220,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
"--dport", strconv.Itoa(svcInfo.Port()),
|
||||
)
|
||||
if proxier.masqueradeAll {
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(internalTrafficChain),
|
||||
args,
|
||||
"-j", string(kubeMarkMasqChain))
|
||||
@ -1220,7 +1230,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// Service range, routing to any node, and that node will
|
||||
// bridge into the Service for you. Since that might bounce
|
||||
// off-node, we masquerade here.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(internalTrafficChain),
|
||||
args,
|
||||
proxier.localDetector.IfNotLocal(),
|
||||
@ -1233,12 +1243,12 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// jump to externalTrafficChain, which will handle some special cases and
|
||||
// then jump to externalPolicyChain.
|
||||
if usesExternalTrafficChain {
|
||||
proxier.natChains.Write(utiliptables.MakeChainLine(externalTrafficChain))
|
||||
natChains.Write(utiliptables.MakeChainLine(externalTrafficChain))
|
||||
|
||||
if !svcInfo.ExternalPolicyLocal() {
|
||||
// If we are using non-local endpoints we need to masquerade,
|
||||
// in case we cross nodes.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(externalTrafficChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"masquerade traffic for %s external destinations"`, svcPortNameString),
|
||||
"-j", string(kubeMarkMasqChain))
|
||||
@ -1251,7 +1261,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// traffic as a special-case. It is subject to neither
|
||||
// form of traffic policy, which simulates going up-and-out
|
||||
// to an external load-balancer and coming back in.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(externalTrafficChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"pod traffic for %s external destinations"`, svcPortNameString),
|
||||
proxier.localDetector.IfLocal(),
|
||||
@ -1261,7 +1271,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// Locally originated traffic (not a pod, but the host node)
|
||||
// still needs masquerade because the LBIP itself is a local
|
||||
// address, so that will be the chosen source IP.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(externalTrafficChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"masquerade LOCAL traffic for %s external destinations"`, svcPortNameString),
|
||||
"-m", "addrtype", "--src-type", "LOCAL",
|
||||
@ -1270,7 +1280,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// Redirect all src-type=LOCAL -> external destination to the
|
||||
// policy=cluster chain. This allows traffic originating
|
||||
// from the host to be redirected to the service correctly.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(externalTrafficChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"route LOCAL traffic for %s external destinations"`, svcPortNameString),
|
||||
"-m", "addrtype", "--src-type", "LOCAL",
|
||||
@ -1279,7 +1289,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
|
||||
// Anything else falls thru to the appropriate policy chain.
|
||||
if hasExternalEndpoints {
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(externalTrafficChain),
|
||||
"-j", string(externalPolicyChain))
|
||||
}
|
||||
@ -1287,7 +1297,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
|
||||
// Set up firewall chain, if needed
|
||||
if usesFWChain {
|
||||
proxier.natChains.Write(utiliptables.MakeChainLine(fwChain))
|
||||
natChains.Write(utiliptables.MakeChainLine(fwChain))
|
||||
|
||||
// The service firewall rules are created based on the
|
||||
// loadBalancerSourceRanges field. This only works for VIP-like
|
||||
@ -1302,7 +1312,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// firewall filter based on each source range
|
||||
allowFromNode := false
|
||||
for _, src := range svcInfo.LoadBalancerSourceRanges() {
|
||||
proxier.natRules.Write(args, "-s", src, "-j", string(externalTrafficChain))
|
||||
natRules.Write(args, "-s", src, "-j", string(externalTrafficChain))
|
||||
_, cidr, err := netutils.ParseCIDRSloppy(src)
|
||||
if err != nil {
|
||||
klog.ErrorS(err, "Error parsing CIDR in LoadBalancerSourceRanges, dropping it", "cidr", cidr)
|
||||
@ -1317,7 +1327,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// need the following rules to allow requests from this node.
|
||||
if allowFromNode {
|
||||
for _, lbip := range svcInfo.LoadBalancerIPStrings() {
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
args,
|
||||
"-s", lbip,
|
||||
"-j", string(externalTrafficChain))
|
||||
@ -1326,7 +1336,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// If the packet was able to reach the end of firewall chain,
|
||||
// then it did not get DNATed, so it will match the
|
||||
// corresponding KUBE-PROXY-FIREWALL rule.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
"-A", string(fwChain),
|
||||
"-m", "comment", "--comment", fmt.Sprintf(`"other traffic to %s will be dropped by KUBE-PROXY-FIREWALL"`, svcPortNameString),
|
||||
)
|
||||
@ -1335,15 +1345,15 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
// If Cluster policy is in use, create the chain and create rules jumping
|
||||
// from clusterPolicyChain to the clusterEndpoints
|
||||
if usesClusterPolicyChain {
|
||||
proxier.natChains.Write(utiliptables.MakeChainLine(clusterPolicyChain))
|
||||
proxier.writeServiceToEndpointRules(svcPortNameString, svcInfo, clusterPolicyChain, clusterEndpoints, args)
|
||||
natChains.Write(utiliptables.MakeChainLine(clusterPolicyChain))
|
||||
proxier.writeServiceToEndpointRules(natRules, svcPortNameString, svcInfo, clusterPolicyChain, clusterEndpoints, args)
|
||||
}
|
||||
|
||||
// If Local policy is in use, create the chain and create rules jumping
|
||||
// from localPolicyChain to the localEndpoints
|
||||
if usesLocalPolicyChain {
|
||||
proxier.natChains.Write(utiliptables.MakeChainLine(localPolicyChain))
|
||||
proxier.writeServiceToEndpointRules(svcPortNameString, svcInfo, localPolicyChain, localEndpoints, args)
|
||||
natChains.Write(utiliptables.MakeChainLine(localPolicyChain))
|
||||
proxier.writeServiceToEndpointRules(natRules, svcPortNameString, svcInfo, localPolicyChain, localEndpoints, args)
|
||||
}
|
||||
|
||||
// Generate the per-endpoint chains.
|
||||
@ -1357,13 +1367,13 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
endpointChain := epInfo.ChainName
|
||||
|
||||
// Create the endpoint chain
|
||||
proxier.natChains.Write(utiliptables.MakeChainLine(endpointChain))
|
||||
natChains.Write(utiliptables.MakeChainLine(endpointChain))
|
||||
activeNATChains[endpointChain] = true
|
||||
|
||||
args = append(args[:0], "-A", string(endpointChain))
|
||||
args = proxier.appendServiceCommentLocked(args, svcPortNameString)
|
||||
// Handle traffic that loops back to the originator with SNAT.
|
||||
proxier.natRules.Write(
|
||||
natRules.Write(
|
||||
args,
|
||||
"-s", epInfo.IP(),
|
||||
"-j", string(kubeMarkMasqChain))
|
||||
@ -1373,7 +1383,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
}
|
||||
// DNAT to final destination.
|
||||
args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", epInfo.Endpoint)
|
||||
proxier.natRules.Write(args)
|
||||
natRules.Write(args)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1483,7 +1493,9 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
)
|
||||
|
||||
metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)).Set(float64(proxier.filterRules.Lines()))
|
||||
metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)).Set(float64(proxier.natRules.Lines() - deletedChains))
|
||||
metrics.IptablesRulesLastSync.WithLabelValues(string(utiliptables.TableFilter)).Set(float64(proxier.filterRules.Lines()))
|
||||
metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)).Set(float64(proxier.natRules.Lines() + skippedNatRules.Lines() - deletedChains))
|
||||
metrics.IptablesRulesLastSync.WithLabelValues(string(utiliptables.TableNAT)).Set(float64(proxier.natRules.Lines() - deletedChains))
|
||||
|
||||
// Sync rules.
|
||||
proxier.iptablesData.Reset()
|
||||
@ -1550,7 +1562,7 @@ func (proxier *Proxier) syncProxyRules() {
|
||||
conntrack.CleanStaleEntries(proxier.iptables.IsIPv6(), proxier.exec, proxier.svcPortMap, serviceUpdateResult, endpointUpdateResult)
|
||||
}
|
||||
|
||||
func (proxier *Proxier) writeServiceToEndpointRules(svcPortNameString string, svcInfo proxy.ServicePort, svcChain utiliptables.Chain, endpoints []proxy.Endpoint, args []string) {
|
||||
func (proxier *Proxier) writeServiceToEndpointRules(natRules *proxyutil.LineBuffer, svcPortNameString string, svcInfo proxy.ServicePort, svcChain utiliptables.Chain, endpoints []proxy.Endpoint, args []string) {
|
||||
// First write session affinity rules, if applicable.
|
||||
if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP {
|
||||
for _, ep := range endpoints {
|
||||
@ -1569,7 +1581,7 @@ func (proxier *Proxier) writeServiceToEndpointRules(svcPortNameString string, sv
|
||||
"--rcheck", "--seconds", strconv.Itoa(svcInfo.StickyMaxAgeSeconds()), "--reap",
|
||||
"-j", string(epInfo.ChainName),
|
||||
)
|
||||
proxier.natRules.Write(args)
|
||||
natRules.Write(args)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1592,6 +1604,6 @@ func (proxier *Proxier) writeServiceToEndpointRules(svcPortNameString string, sv
|
||||
"--probability", proxier.probability(numEndpoints-i))
|
||||
}
|
||||
// The final (or only if n == 1) rule is a guaranteed match.
|
||||
proxier.natRules.Write(args, "-j", string(epInfo.ChainName))
|
||||
natRules.Write(args, "-j", string(epInfo.ChainName))
|
||||
}
|
||||
}
|
||||
|
||||
@ -613,6 +613,15 @@ func countRulesFromMetric(tableName utiliptables.Table) int {
|
||||
return int(numRulesFloat)
|
||||
}
|
||||
|
||||
func countRulesFromLastSyncMetric(tableName utiliptables.Table) int {
|
||||
numRulesFloat, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesLastSync.WithLabelValues(string(tableName)))
|
||||
if err != nil {
|
||||
klog.ErrorS(err, "metrics are not registered?")
|
||||
return -1
|
||||
}
|
||||
return int(numRulesFloat)
|
||||
}
|
||||
|
||||
// findAllMatches takes an array of lines and a pattern with one parenthesized group, and
|
||||
// returns a sorted array of all of the unique matches of the parenthesized group.
|
||||
func findAllMatches(lines []string, pattern string) []string {
|
||||
@ -7694,11 +7703,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), true, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced := countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric := countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric := countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
rulesTotal := rulesSynced
|
||||
rulesTotalMetric := countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
// Add a new service and its endpoints. (This will only sync the SVC and SEP rules
|
||||
// for the new service, not the existing ones.)
|
||||
makeServiceMap(fp,
|
||||
@ -7766,11 +7781,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// We added 1 KUBE-SERVICES rule, 2 KUBE-SVC-X27LE4BHSL4DOUIK rules, and 2
|
||||
// KUBE-SEP-BSWRHOQ77KEXZLNL rules.
|
||||
rulesTotal += 5
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
// Delete a service. (Won't update the other services.)
|
||||
fp.OnServiceDelete(svc2)
|
||||
fp.syncProxyRules()
|
||||
@ -7809,11 +7832,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// We deleted 1 KUBE-SERVICES rule, 2 KUBE-SVC-2VJB64SDSIJUP5T6 rules, and 2
|
||||
// KUBE-SEP-UHEGFW77JX3KXTOV rules
|
||||
rulesTotal -= 5
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
// Add a service, sync, then add its endpoints. (The first sync will be a no-op other
|
||||
// than adding the REJECT rule. The second sync will create the new service.)
|
||||
var svc4 *v1.Service
|
||||
@ -7861,11 +7892,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// The REJECT rule is in "filter", not NAT, so the number of NAT rules hasn't
|
||||
// changed.
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
populateEndpointSlices(fp,
|
||||
makeTestEndpointSlice("ns4", "svc4", 1, func(eps *discovery.EndpointSlice) {
|
||||
eps.AddressType = discovery.AddressTypeIPv4
|
||||
@ -7917,11 +7955,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// We added 1 KUBE-SERVICES rule, 2 KUBE-SVC-4SW47YFZTEDKD3PK rules, and
|
||||
// 2 KUBE-SEP-AYCN5HPXMIRJNJXU rules
|
||||
rulesTotal += 5
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
// Change an endpoint of an existing service. This will cause its SVC and SEP
|
||||
// chains to be rewritten.
|
||||
eps3update := eps3.DeepCopy()
|
||||
@ -7968,11 +8014,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// We rewrote existing rules but did not change the overall number of rules.
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
// Add an endpoint to a service. This will cause its SVC and SEP chains to be rewritten.
|
||||
eps3update2 := eps3update.DeepCopy()
|
||||
eps3update2.Endpoints = append(eps3update2.Endpoints, discovery.Endpoint{Addresses: []string{"10.0.3.3"}})
|
||||
@ -8020,11 +8072,20 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// We added 2 KUBE-SEP-JVVZVJ7BSEPPRNBS rules and 1 KUBE-SVC-X27LE4BHSL4DOUIK rule
|
||||
// jumping to the new SEP chain. The other rules related to svc3 got rewritten,
|
||||
// but that does not change the count of rules.
|
||||
rulesTotal += 3
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
// Sync with no new changes... This will not rewrite any SVC or SEP chains
|
||||
fp.syncProxyRules()
|
||||
|
||||
@ -8059,11 +8120,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// (No changes)
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
|
||||
// Now force a partial resync error and ensure that it recovers correctly
|
||||
if fp.needFullSync {
|
||||
t.Fatalf("Proxier unexpectedly already needs a full sync?")
|
||||
@ -8163,10 +8230,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) {
|
||||
assertIPTablesRulesEqual(t, getLine(), false, expected, fp.iptablesData.String())
|
||||
|
||||
rulesSynced = countRules(utiliptables.TableNAT, expected)
|
||||
rulesSyncedMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
rulesSyncedMetric = countRulesFromLastSyncMetric(utiliptables.TableNAT)
|
||||
if rulesSyncedMetric != rulesSynced {
|
||||
t.Errorf("metric shows %d rules synced but iptables data shows %d", rulesSyncedMetric, rulesSynced)
|
||||
}
|
||||
|
||||
// We deleted 1 KUBE-SERVICES rule, 2 KUBE-SVC-4SW47YFZTEDKD3PK rules, and 2
|
||||
// KUBE-SEP-AYCN5HPXMIRJNJXU rules
|
||||
rulesTotal -= 5
|
||||
rulesTotalMetric = countRulesFromMetric(utiliptables.TableNAT)
|
||||
if rulesTotalMetric != rulesTotal {
|
||||
t.Errorf("metric shows %d rules total but expected %d", rulesTotalMetric, rulesTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoEndpointsMetric(t *testing.T) {
|
||||
|
||||
@ -160,12 +160,25 @@ var (
|
||||
},
|
||||
)
|
||||
|
||||
// IptablesRulesTotal is the number of iptables rules that the iptables proxy installs.
|
||||
// IptablesRulesTotal is the total number of iptables rules that the iptables
|
||||
// proxy has installed.
|
||||
IptablesRulesTotal = metrics.NewGaugeVec(
|
||||
&metrics.GaugeOpts{
|
||||
Subsystem: kubeProxySubsystem,
|
||||
Name: "sync_proxy_rules_iptables_total",
|
||||
Help: "Number of proxy iptables rules programmed",
|
||||
Help: "Total number of iptables rules owned by kube-proxy",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"table"},
|
||||
)
|
||||
|
||||
// IptablesRulesLastSync is the number of iptables rules that the iptables proxy
|
||||
// updated in the last sync.
|
||||
IptablesRulesLastSync = metrics.NewGaugeVec(
|
||||
&metrics.GaugeOpts{
|
||||
Subsystem: kubeProxySubsystem,
|
||||
Name: "sync_proxy_rules_iptables_last",
|
||||
Help: "Number of iptables rules written by kube-proxy in last sync",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"table"},
|
||||
@ -212,6 +225,7 @@ func RegisterMetrics() {
|
||||
legacyregistry.MustRegister(ServiceChangesPending)
|
||||
legacyregistry.MustRegister(ServiceChangesTotal)
|
||||
legacyregistry.MustRegister(IptablesRulesTotal)
|
||||
legacyregistry.MustRegister(IptablesRulesLastSync)
|
||||
legacyregistry.MustRegister(IptablesRestoreFailuresTotal)
|
||||
legacyregistry.MustRegister(IptablesPartialRestoreFailuresTotal)
|
||||
legacyregistry.MustRegister(SyncProxyRulesLastQueuedTimestamp)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user