From 2e1a884bf3dd78b887c14c3f4a743314755105d8 Mon Sep 17 00:00:00 2001
From: Casey Callendrello <cdc@redhat.com>
Date: Wed, 15 Apr 2020 16:40:45 +0200
Subject: [PATCH] pkg/proxy: add last-queued-timestamp metric

This adds a metric, kubeproxy_sync_proxy_rules_last_queued_timestamp,
that captures the last time a change was queued to be applied to the
proxy. This matches the healthz logic, which fails if a pending change
is stale.

This allows us to write alerts that mirror healthz.

Signed-off-by: Casey Callendrello <cdc@redhat.com>
---
 pkg/proxy/iptables/proxier.go  |  1 +
 pkg/proxy/ipvs/proxier.go      |  1 +
 pkg/proxy/metrics/metrics.go   | 13 +++++++++++++
 pkg/proxy/winkernel/proxier.go |  1 +
 4 files changed, 16 insertions(+)

diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go
index 4344319482d..5ea5fb1adbf 100644
--- a/pkg/proxy/iptables/proxier.go
+++ b/pkg/proxy/iptables/proxier.go
@@ -504,6 +504,7 @@ func (proxier *Proxier) probability(n int) string {
 func (proxier *Proxier) Sync() {
 	if proxier.healthzServer != nil {
 		proxier.healthzServer.QueuedUpdate()
+		metrics.SyncProxyRulesLastQueuedTimestamp.SetToCurrentTime()
 	}
 	proxier.syncRunner.Run()
 }
diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index 9f571e446a2..9515ad0603e 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -810,6 +810,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 func (proxier *Proxier) Sync() {
 	if proxier.healthzServer != nil {
 		proxier.healthzServer.QueuedUpdate()
+		metrics.SyncProxyRulesLastQueuedTimestamp.SetToCurrentTime()
 	}
 	proxier.syncRunner.Run()
 }
diff --git a/pkg/proxy/metrics/metrics.go b/pkg/proxy/metrics/metrics.go
index 31fecbd2a14..e917704c7c5 100644
--- a/pkg/proxy/metrics/metrics.go
+++ b/pkg/proxy/metrics/metrics.go
@@ -125,6 +125,18 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 	)
+
+	// SyncProxyRulesLastQueuedTimestamp is the last time a proxy sync was
+	// requested. If this is much larger than
+	// kubeproxy_sync_proxy_rules_last_timestamp_seconds, then something is hung.
+	SyncProxyRulesLastQueuedTimestamp = metrics.NewGauge(
+		&metrics.GaugeOpts{
+			Subsystem:      kubeProxySubsystem,
+			Name:           "sync_proxy_rules_last_queued_timestamp_seconds",
+			Help:           "The last time a sync of proxy rules was queued",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
 )
 
 var registerMetricsOnce sync.Once
@@ -140,6 +152,7 @@ func RegisterMetrics() {
 		legacyregistry.MustRegister(ServiceChangesPending)
 		legacyregistry.MustRegister(ServiceChangesTotal)
 		legacyregistry.MustRegister(IptablesRestoreFailuresTotal)
+		legacyregistry.MustRegister(SyncProxyRulesLastQueuedTimestamp)
 	})
 }
 
diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go
index 3d742f43646..40adcd59188 100644
--- a/pkg/proxy/winkernel/proxier.go
+++ b/pkg/proxy/winkernel/proxier.go
@@ -736,6 +736,7 @@ func getHnsNetworkInfo(hnsNetworkName string) (*hnsNetworkInfo, error) {
 func (proxier *Proxier) Sync() {
 	if proxier.healthzServer != nil {
 		proxier.healthzServer.QueuedUpdate()
+		metrics.SyncProxyRulesLastQueuedTimestamp.SetToCurrentTime()
 	}
 	proxier.syncRunner.Run()
 }