github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 23:37:01 +00:00
Code Issues Projects Releases Wiki Activity

add failopen metric

Browse Source
This commit is contained in:
Luigi Tagliamonte 2021-12-21 14:11:12 -08:00
parent 38ec822b46
commit 6542f4bb99
3 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go
View File

@ -116,6 +116,7 @@ type AdmissionMetrics struct {
controller *metricSet
webhook *metricSet
webhookRejection *metrics.CounterVec
webhookFailOpen *metrics.CounterVec
webhookRequest *metrics.CounterVec
}
@ -196,6 +197,16 @@ func newAdmissionMetrics() *AdmissionMetrics {
},
[]string{"name", "type", "operation", "error_type", "rejection_code"})
webhookFailOpen := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "webhook_fail_open_count",
Help: "Admission webhook fail open count, identified by name and broken out for each admission type (validating or mutating).",
StabilityLevel: metrics.ALPHA,
},
[]string{"name", "type"})
webhookRequest := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
@ -210,8 +221,9 @@ func newAdmissionMetrics() *AdmissionMetrics {
controller.mustRegister()
webhook.mustRegister()
legacyregistry.MustRegister(webhookRejection)
legacyregistry.MustRegister(webhookFailOpen)
legacyregistry.MustRegister(webhookRequest)
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookRequest: webhookRequest}
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest}
}
func (m *AdmissionMetrics) reset() {
@ -250,6 +262,11 @@ func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, st
m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc()
}
// ObserveWebhookFailOpen records validating or mutating webhook that fail open.
func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, stepType string) {
m.webhookFailOpen.WithContext(ctx).WithLabelValues(name, stepType).Inc()
}
type metricSet struct {
latencies *metrics.HistogramVec
latenciesSummary *metrics.SummaryVec

2
staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go
View File

@ -178,7 +178,7 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
annotator.addFailedOpenAnnotation()
utilruntime.HandleError(callErr)

2
staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go
View File

@ -140,7 +140,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
value := hook.Name
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {