github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 19:01:49 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #107171 from ltagliamonte-dd/add_failopen_metrics

Browse Source 
add failopen metric
This commit is contained in:
Kubernetes Prow Robot 2022-01-07 03:02:29 -08:00 committed by GitHub
commit 9d0d2e8ece
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go
View File

@ -116,6 +116,7 @@ type AdmissionMetrics struct {
controller *metricSet
webhook *metricSet
webhookRejection *metrics.CounterVec
webhookFailOpen *metrics.CounterVec
webhookRequest *metrics.CounterVec
}
@ -196,6 +197,16 @@ func newAdmissionMetrics() *AdmissionMetrics {
},
[]string{"name", "type", "operation", "error_type", "rejection_code"})
webhookFailOpen := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "webhook_fail_open_count",
Help: "Admission webhook fail open count, identified by name and broken out for each admission type (validating or mutating).",
StabilityLevel: metrics.ALPHA,
},
[]string{"name", "type"})
webhookRequest := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
@ -210,8 +221,9 @@ func newAdmissionMetrics() *AdmissionMetrics {
controller.mustRegister()
webhook.mustRegister()
legacyregistry.MustRegister(webhookRejection)
legacyregistry.MustRegister(webhookFailOpen)
legacyregistry.MustRegister(webhookRequest)
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookRequest: webhookRequest}
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest}
}
func (m *AdmissionMetrics) reset() {
@ -250,6 +262,11 @@ func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, st
m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc()
}
// ObserveWebhookFailOpen records validating or mutating webhook that fail open.
func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, stepType string) {
m.webhookFailOpen.WithContext(ctx).WithLabelValues(name, stepType).Inc()
}
type metricSet struct {
latencies *metrics.HistogramVec
latenciesSummary *metrics.SummaryVec

17
staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go
View File

@ -159,6 +159,23 @@ func TestObserveWebhookRejection(t *testing.T) {
expectCounterValue(t, "apiserver_admission_webhook_rejection_count", wantLabelsAPIServerInternalError, 1)
}
func TestObserveWebhookFailOpen(t *testing.T) {
defer Metrics.reset()
defer legacyregistry.Reset()
Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepAdmit)
Metrics.ObserveWebhookFailOpen(context.TODO(), "x", stepValidate)
wantLabelsCounterAdmit := map[string]string{
"name": "x",
"type": "admit",
}
wantLabelsCounterValidate := map[string]string{
"name": "x",
"type": "validate",
}
expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterAdmit, 1)
expectCounterValue(t, "apiserver_admission_webhook_fail_open_count", wantLabelsCounterValidate, 1)
}
func TestWithMetrics(t *testing.T) {
defer Metrics.reset()
defer legacyregistry.Reset()

2
staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go
View File

@ -178,7 +178,7 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
annotator.addFailedOpenAnnotation()
utilruntime.HandleError(callErr)

2
staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go
View File

@ -140,7 +140,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
value := hook.Name
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {