mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Merge pull request #107171 from ltagliamonte-dd/add_failopen_metrics
add failopen metric
This commit is contained in:
commit
9d0d2e8ece
@ -116,6 +116,7 @@ type AdmissionMetrics struct {
|
|||||||
controller *metricSet
|
controller *metricSet
|
||||||
webhook *metricSet
|
webhook *metricSet
|
||||||
webhookRejection *metrics.CounterVec
|
webhookRejection *metrics.CounterVec
|
||||||
|
webhookFailOpen *metrics.CounterVec
|
||||||
webhookRequest *metrics.CounterVec
|
webhookRequest *metrics.CounterVec
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -196,6 +197,16 @@ func newAdmissionMetrics() *AdmissionMetrics {
|
|||||||
},
|
},
|
||||||
[]string{"name", "type", "operation", "error_type", "rejection_code"})
|
[]string{"name", "type", "operation", "error_type", "rejection_code"})
|
||||||
|
|
||||||
|
webhookFailOpen := metrics.NewCounterVec(
|
||||||
|
&metrics.CounterOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "webhook_fail_open_count",
|
||||||
|
Help: "Admission webhook fail open count, identified by name and broken out for each admission type (validating or mutating).",
|
||||||
|
StabilityLevel: metrics.ALPHA,
|
||||||
|
},
|
||||||
|
[]string{"name", "type"})
|
||||||
|
|
||||||
webhookRequest := metrics.NewCounterVec(
|
webhookRequest := metrics.NewCounterVec(
|
||||||
&metrics.CounterOpts{
|
&metrics.CounterOpts{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
@ -210,8 +221,9 @@ func newAdmissionMetrics() *AdmissionMetrics {
|
|||||||
controller.mustRegister()
|
controller.mustRegister()
|
||||||
webhook.mustRegister()
|
webhook.mustRegister()
|
||||||
legacyregistry.MustRegister(webhookRejection)
|
legacyregistry.MustRegister(webhookRejection)
|
||||||
|
legacyregistry.MustRegister(webhookFailOpen)
|
||||||
legacyregistry.MustRegister(webhookRequest)
|
legacyregistry.MustRegister(webhookRequest)
|
||||||
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookRequest: webhookRequest}
|
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *AdmissionMetrics) reset() {
|
func (m *AdmissionMetrics) reset() {
|
||||||
@ -250,6 +262,11 @@ func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, st
|
|||||||
m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc()
|
m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ObserveWebhookFailOpen records validating or mutating webhook that fail open.
|
||||||
|
func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, stepType string) {
|
||||||
|
m.webhookFailOpen.WithContext(ctx).WithLabelValues(name, stepType).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
type metricSet struct {
|
type metricSet struct {
|
||||||
latencies *metrics.HistogramVec
|
latencies *metrics.HistogramVec
|
||||||
latenciesSummary *metrics.SummaryVec
|
latenciesSummary *metrics.SummaryVec
|
||||||
|
@ -159,6 +159,23 @@ func TestObserveWebhookRejection(t *testing.T) {
|
|||||||
expectCounterValue(t, "apiserver_admission_webhook_rejection_count", wantLabelsAPIServerInternalError, 1)
|
expectCounterValue(t, "apiserver_admission_webhook_rejection_count", wantLabelsAPIServerInternalError, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestObserveWebhookFailOpen(t *testing.T) {
|
||||||
|
defer Metrics.reset()
|
||||||
|
defer legacyregistry.Reset()
|
||||||
|
Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepAdmit)
|
||||||
|
Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepValidate)
|
||||||
|
wantLabelsCounterAdmit := map[string]string{
|
||||||
|
"name": "x",
|
||||||
|
"type": "admit",
|
||||||
|
}
|
||||||
|
wantLabelsCounterValidate := map[string]string{
|
||||||
|
"name": "x",
|
||||||
|
"type": "validate",
|
||||||
|
}
|
||||||
|
expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterAdmit, 1)
|
||||||
|
expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterValidate, 1)
|
||||||
|
}
|
||||||
|
|
||||||
func TestWithMetrics(t *testing.T) {
|
func TestWithMetrics(t *testing.T) {
|
||||||
defer Metrics.reset()
|
defer Metrics.reset()
|
||||||
defer legacyregistry.Reset()
|
defer legacyregistry.Reset()
|
||||||
|
@ -178,7 +178,7 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
|
|||||||
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
|
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
|
||||||
if ignoreClientCallFailures {
|
if ignoreClientCallFailures {
|
||||||
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
|
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
|
||||||
|
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
|
||||||
annotator.addFailedOpenAnnotation()
|
annotator.addFailedOpenAnnotation()
|
||||||
|
|
||||||
utilruntime.HandleError(callErr)
|
utilruntime.HandleError(callErr)
|
||||||
|
@ -140,7 +140,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
|
|||||||
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
|
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
|
||||||
if ignoreClientCallFailures {
|
if ignoreClientCallFailures {
|
||||||
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
|
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
|
||||||
|
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
|
||||||
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
|
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
|
||||||
value := hook.Name
|
value := hook.Name
|
||||||
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {
|
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {
|
||||||
|
Loading…
Reference in New Issue
Block a user