From 1a5e77135b2c35c17dea3a85b753ab4a5f133387 Mon Sep 17 00:00:00 2001 From: Lalit Chauhan Date: Mon, 8 Sep 2025 18:48:04 +0000 Subject: [PATCH] Add fine grained metrics to narrow down DV mismatches and panics --- .../certificates/certificates/strategy.go | 20 ++++--- .../replicationcontroller/storage/storage.go | 5 +- .../core/replicationcontroller/strategy.go | 10 ++-- .../apiserver/pkg/registry/rest/validate.go | 24 +++++--- .../pkg/registry/rest/validate_test.go | 6 +- .../apiserver/pkg/validation/metrics.go | 42 ++++++++++--- .../apiserver/pkg/validation/metrics_test.go | 59 ++++++++++++++----- 7 files changed, 118 insertions(+), 48 deletions(-) diff --git a/pkg/registry/certificates/certificates/strategy.go b/pkg/registry/certificates/certificates/strategy.go index fde7fa70fe3..d910548a91d 100644 --- a/pkg/registry/certificates/certificates/strategy.go +++ b/pkg/registry/certificates/certificates/strategy.go @@ -122,11 +122,12 @@ func (csrStrategy) Validate(ctx context.Context, obj runtime.Object) field.Error // Determine if takeover is enabled takeover := utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidationTakeover) + const validationIdentifier = "csr_create" // Run declarative validation with panic recovery - declarativeErrs := rest.ValidateDeclaratively(ctx, legacyscheme.Scheme, csr, rest.WithTakeover(takeover)) + declarativeErrs := rest.ValidateDeclaratively(ctx, legacyscheme.Scheme, csr, rest.WithTakeover(takeover), rest.WithValidationIdentifier(validationIdentifier)) // Compare imperative and declarative errors and log + emit metric if there's a mismatch - rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, allErrs, declarativeErrs, takeover) + rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, allErrs, declarativeErrs, takeover, validationIdentifier) // Only apply declarative errors if takeover is enabled if takeover { @@ -152,11 +153,12 @@ func (csrStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) // Determine if takeover is enabled takeover := utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidationTakeover) + const validationIdentifier = "csr_update" // Run declarative update validation with panic recovery - declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newCSR, oldCSR, rest.WithTakeover(takeover)) + declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newCSR, oldCSR, rest.WithTakeover(takeover), rest.WithValidationIdentifier(validationIdentifier)) // Compare imperative and declarative errors and emit metric if there's a mismatch - rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover) + rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover, validationIdentifier) // Only apply declarative errors if takeover is enabled if takeover { @@ -284,11 +286,12 @@ func (csrStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Ob // Determine if takeover is enabled takeover := utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidationTakeover) + const validationIdentifier = "csr_status_update" // Run declarative update validation with panic recovery - declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newCSR, oldCSR, rest.WithTakeover(takeover)) + declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newCSR, oldCSR, rest.WithTakeover(takeover), rest.WithValidationIdentifier(validationIdentifier)) // Compare imperative and declarative errors and emit metric if there's a mismatch - rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover) + rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover, validationIdentifier) // Only apply declarative errors if takeover is enabled if takeover { @@ -354,11 +357,12 @@ func (csrApprovalStrategy) ValidateUpdate(ctx context.Context, obj, old runtime. // Determine if takeover is enabled takeover := utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidationTakeover) + const validationIdentifier = "csr_approval_update" // Run declarative update validation with panic recovery - declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newCSR, oldCSR, rest.WithTakeover(takeover)) + declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newCSR, oldCSR, rest.WithTakeover(takeover), rest.WithValidationIdentifier(validationIdentifier)) // Compare imperative and declarative errors and emit metric if there's a mismatch - rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover) + rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover, validationIdentifier) // Only apply declarative errors if takeover is enabled if takeover { diff --git a/pkg/registry/core/replicationcontroller/storage/storage.go b/pkg/registry/core/replicationcontroller/storage/storage.go index a43c700c4c1..2d1e99e5719 100644 --- a/pkg/registry/core/replicationcontroller/storage/storage.go +++ b/pkg/registry/core/replicationcontroller/storage/storage.go @@ -321,12 +321,13 @@ func (i *scaleUpdatedObjectInfo) UpdatedObject(ctx context.Context, oldObj runti // Determine if takeover is enabled takeover := utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidationTakeover) + const validationIdentifier = "rc_scale" // Run declarative validation with panic recovery declarativeErrs := rest.ValidateUpdateDeclaratively( - ctx, legacyscheme.Scheme, scale, oldScale, rest.WithTakeover(takeover), rest.WithSubresourceMapper(i.scaleGVKMapper)) + ctx, legacyscheme.Scheme, scale, oldScale, rest.WithTakeover(takeover), rest.WithSubresourceMapper(i.scaleGVKMapper), rest.WithValidationIdentifier(validationIdentifier)) // Compare imperative and declarative errors and log + emit metric if there's a mismatch - rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover) + rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover, validationIdentifier) // Only apply declarative errors if takeover is enabled if takeover { diff --git a/pkg/registry/core/replicationcontroller/strategy.go b/pkg/registry/core/replicationcontroller/strategy.go index b2a5c85f36e..b86c7c13cde 100644 --- a/pkg/registry/core/replicationcontroller/strategy.go +++ b/pkg/registry/core/replicationcontroller/strategy.go @@ -134,12 +134,13 @@ func (rcStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorL if utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidation) { // Determine if takeover is enabled takeover := utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidationTakeover) + const validationIdentifier = "rc_create" // Run declarative validation with panic recovery - declarativeErrs := rest.ValidateDeclaratively(ctx, legacyscheme.Scheme, controller, rest.WithTakeover(takeover)) + declarativeErrs := rest.ValidateDeclaratively(ctx, legacyscheme.Scheme, controller, rest.WithTakeover(takeover), rest.WithValidationIdentifier(validationIdentifier)) // Compare imperative and declarative errors and log + emit metric if there's a mismatch - rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, allErrs, declarativeErrs, takeover) + rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, allErrs, declarativeErrs, takeover, validationIdentifier) // Only apply declarative errors if takeover is enabled if takeover { @@ -202,12 +203,13 @@ func (rcStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) f if utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidation) { // Determine if takeover is enabled takeover := utilfeature.DefaultFeatureGate.Enabled(features.DeclarativeValidationTakeover) + const validationIdentifier = "rc_update" // Run declarative update validation with panic recovery - declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newRc, oldRc, rest.WithTakeover(takeover)) + declarativeErrs := rest.ValidateUpdateDeclaratively(ctx, legacyscheme.Scheme, newRc, oldRc, rest.WithTakeover(takeover), rest.WithValidationIdentifier(validationIdentifier)) // Compare imperative and declarative errors and emit metric if there's a mismatch - rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover) + rest.CompareDeclarativeErrorsAndEmitMismatches(ctx, errs, declarativeErrs, takeover, validationIdentifier) // Only apply declarative errors if takeover is enabled if takeover { diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/validate.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/validate.go index c220fa93b7a..112377a2d8a 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/validate.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/validate.go @@ -51,6 +51,13 @@ func WithTakeover(takeover bool) ValidationConfig { } } +// WithValidationIdentifier sets the validation identifier, which is used to determine the source of a mismatch in metrics. +func WithValidationIdentifier(identifier string) ValidationConfig { + return func(config *validationConfigOption) { + config.validationIdentifier = identifier + } +} + // WithSubresourceMapper sets the subresource mapper for validation. // This should be used when registering validation for polymorphic subresources like /scale. // @@ -79,6 +86,7 @@ type validationConfigOption struct { options []string takeover bool subresourceGVKMapper GroupVersionKindProvider + validationIdentifier string } // ValidateDeclaratively validates obj against declarative validation tags @@ -99,7 +107,7 @@ func ValidateDeclaratively(ctx context.Context, scheme *runtime.Scheme, obj runt o(cfg) } - return panicSafeValidateFunc(validateDeclaratively, cfg.takeover)(ctx, scheme, obj, nil, cfg) + return panicSafeValidateFunc(validateDeclaratively, cfg.takeover, cfg.validationIdentifier)(ctx, scheme, obj, nil, cfg) } // ValidateUpdateDeclaratively validates obj and oldObj against declarative @@ -119,7 +127,7 @@ func ValidateUpdateDeclaratively(ctx context.Context, scheme *runtime.Scheme, ob for _, o := range configOpts { o(cfg) } - return panicSafeValidateFunc(validateDeclaratively, cfg.takeover)(ctx, scheme, obj, oldObj, cfg) + return panicSafeValidateFunc(validateDeclaratively, cfg.takeover, cfg.validationIdentifier)(ctx, scheme, obj, oldObj, cfg) } func validateDeclaratively(ctx context.Context, scheme *runtime.Scheme, obj, oldObj runtime.Object, o *validationConfigOption) field.ErrorList { @@ -180,7 +188,7 @@ func parseSubresourcePath(subresourcePath string) ([]string, error) { // CompareDeclarativeErrorsAndEmitMismatches checks for mismatches between imperative and declarative validation // and logs + emits metrics when inconsistencies are found -func CompareDeclarativeErrorsAndEmitMismatches(ctx context.Context, imperativeErrs, declarativeErrs field.ErrorList, takeover bool) { +func CompareDeclarativeErrorsAndEmitMismatches(ctx context.Context, imperativeErrs, declarativeErrs field.ErrorList, takeover bool, validationIdentifier string) { logger := klog.FromContext(ctx) mismatchDetails := gatherDeclarativeValidationMismatches(imperativeErrs, declarativeErrs, takeover) for _, detail := range mismatchDetails { @@ -188,7 +196,7 @@ func CompareDeclarativeErrorsAndEmitMismatches(ctx context.Context, imperativeEr logger.Error(nil, detail) // Increment the metric for the mismatch - validationmetrics.Metrics.IncDeclarativeValidationMismatchMetric() + validationmetrics.Metrics.IncDeclarativeValidationMismatchMetric(validationIdentifier) } } @@ -288,12 +296,12 @@ func gatherDeclarativeValidationMismatches(imperativeErrs, declarativeErrs field // createDeclarativeValidationPanicHandler returns a function with panic recovery logic // that will increment the panic metric and either log or append errors based on the takeover parameter. -func createDeclarativeValidationPanicHandler(ctx context.Context, errs *field.ErrorList, takeover bool) func() { +func createDeclarativeValidationPanicHandler(ctx context.Context, errs *field.ErrorList, takeover bool, validationIdentifier string) func() { logger := klog.FromContext(ctx) return func() { if r := recover(); r != nil { // Increment the panic metric counter - validationmetrics.Metrics.IncDeclarativeValidationPanicMetric() + validationmetrics.Metrics.IncDeclarativeValidationPanicMetric(validationIdentifier) const errorFmt = "panic during declarative validation: %v" if takeover { @@ -313,10 +321,10 @@ func createDeclarativeValidationPanicHandler(ctx context.Context, errs *field.Er // if takeover=false, and adding a validation error if takeover=true. func panicSafeValidateFunc( validateUpdateFunc func(ctx context.Context, scheme *runtime.Scheme, obj, oldObj runtime.Object, o *validationConfigOption) field.ErrorList, - takeover bool, + takeover bool, validationIdentifier string, ) func(ctx context.Context, scheme *runtime.Scheme, obj, oldObj runtime.Object, o *validationConfigOption) field.ErrorList { return func(ctx context.Context, scheme *runtime.Scheme, obj, oldObj runtime.Object, o *validationConfigOption) (errs field.ErrorList) { - defer createDeclarativeValidationPanicHandler(ctx, &errs, takeover)() + defer createDeclarativeValidationPanicHandler(ctx, &errs, takeover, validationIdentifier)() return validateUpdateFunc(ctx, scheme, obj, oldObj, o) } diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/validate_test.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/validate_test.go index cd2d77877cf..160138aa312 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/validate_test.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/validate_test.go @@ -425,7 +425,7 @@ func TestCompareDeclarativeErrorsAndEmitMismatches(t *testing.T) { defer klog.LogToStderr(true) ctx := context.Background() - CompareDeclarativeErrorsAndEmitMismatches(ctx, tc.imperativeErrs, tc.declarativeErrs, tc.takeover) + CompareDeclarativeErrorsAndEmitMismatches(ctx, tc.imperativeErrs, tc.declarativeErrs, tc.takeover, "test_validationIdentifier") klog.Flush() logOutput := buf.String() @@ -511,7 +511,7 @@ func TestWithRecover(t *testing.T) { defer klog.LogToStderr(true) // Pass the takeover flag to panicSafeValidateFunc instead of relying on the feature gate - wrapped := panicSafeValidateFunc(tc.validateFn, tc.takeoverEnabled) + wrapped := panicSafeValidateFunc(tc.validateFn, tc.takeoverEnabled, "test_validationIdentifier") gotErrs := wrapped(ctx, scheme, obj, nil, &validationConfigOption{opType: operation.Create, options: options, takeover: tc.takeoverEnabled}) klog.Flush() @@ -605,7 +605,7 @@ func TestWithRecoverUpdate(t *testing.T) { defer klog.LogToStderr(true) // Pass the takeover flag to panicSafeValidateUpdateFunc instead of relying on the feature gate - wrapped := panicSafeValidateFunc(tc.validateFn, tc.takeoverEnabled) + wrapped := panicSafeValidateFunc(tc.validateFn, tc.takeoverEnabled, "test_validationIdentifier") gotErrs := wrapped(ctx, scheme, obj, oldObj, &validationConfigOption{opType: operation.Update, options: options, takeover: tc.takeoverEnabled}) klog.Flush() diff --git a/staging/src/k8s.io/apiserver/pkg/validation/metrics.go b/staging/src/k8s.io/apiserver/pkg/validation/metrics.go index 0e520cf743f..4d06bc96d67 100644 --- a/staging/src/k8s.io/apiserver/pkg/validation/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/validation/metrics.go @@ -28,8 +28,8 @@ const ( // ValidationMetrics is the interface for validation metrics. type ValidationMetrics interface { - IncDeclarativeValidationMismatchMetric() - IncDeclarativeValidationPanicMetric() + IncDeclarativeValidationMismatchMetric(validationIdentifier string) + IncDeclarativeValidationPanicMetric(validationIdentifier string) IncDuplicateValidationErrorMetric() Reset() } @@ -44,6 +44,16 @@ var validationMetricsInstance = &validationMetrics{ StabilityLevel: metrics.BETA, }, ), + DeclarativeValidationMismatchCounterVector: metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "declarative_validation_parity_discrepancies_total", + Help: "Number of discrepancies between declarative and handwritten validation, broken down by validation identifier.", + StabilityLevel: metrics.ALPHA, + }, + []string{"validation_identifier"}, + ), DeclarativeValidationPanicCounter: metrics.NewCounter( &metrics.CounterOpts{ Namespace: namespace, @@ -53,6 +63,16 @@ var validationMetricsInstance = &validationMetrics{ StabilityLevel: metrics.BETA, }, ), + DeclarativeValidationPanicCounterVector: metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "declarative_validation_panics_total", + Help: "Number of panics in declarative validation, broken down by validation identifier.", + StabilityLevel: metrics.ALPHA, + }, + []string{"validation_identifier"}, + ), DuplicateValidationErrorCounter: metrics.NewCounter( &metrics.CounterOpts{ Namespace: namespace, @@ -69,31 +89,39 @@ var Metrics ValidationMetrics = validationMetricsInstance func init() { legacyregistry.MustRegister(validationMetricsInstance.DeclarativeValidationMismatchCounter) + legacyregistry.MustRegister(validationMetricsInstance.DeclarativeValidationMismatchCounterVector) legacyregistry.MustRegister(validationMetricsInstance.DeclarativeValidationPanicCounter) + legacyregistry.MustRegister(validationMetricsInstance.DeclarativeValidationPanicCounterVector) legacyregistry.MustRegister(validationMetricsInstance.DuplicateValidationErrorCounter) } type validationMetrics struct { - DeclarativeValidationMismatchCounter *metrics.Counter - DeclarativeValidationPanicCounter *metrics.Counter - DuplicateValidationErrorCounter *metrics.Counter + DeclarativeValidationMismatchCounter *metrics.Counter + DeclarativeValidationMismatchCounterVector *metrics.CounterVec + DeclarativeValidationPanicCounter *metrics.Counter + DeclarativeValidationPanicCounterVector *metrics.CounterVec + DuplicateValidationErrorCounter *metrics.Counter } // Reset resets the validation metrics. func (m *validationMetrics) Reset() { m.DeclarativeValidationMismatchCounter.Reset() + m.DeclarativeValidationMismatchCounterVector.Reset() m.DeclarativeValidationPanicCounter.Reset() + m.DeclarativeValidationPanicCounterVector.Reset() m.DuplicateValidationErrorCounter.Reset() } // IncDeclarativeValidationMismatchMetric increments the counter for the declarative_validation_mismatch_total metric. -func (m *validationMetrics) IncDeclarativeValidationMismatchMetric() { +func (m *validationMetrics) IncDeclarativeValidationMismatchMetric(validationIdentifier string) { m.DeclarativeValidationMismatchCounter.Inc() + m.DeclarativeValidationMismatchCounterVector.WithLabelValues(validationIdentifier).Inc() } // IncDeclarativeValidationPanicMetric increments the counter for the declarative_validation_panic_total metric. -func (m *validationMetrics) IncDeclarativeValidationPanicMetric() { +func (m *validationMetrics) IncDeclarativeValidationPanicMetric(validationIdentifier string) { m.DeclarativeValidationPanicCounter.Inc() + m.DeclarativeValidationPanicCounterVector.WithLabelValues(validationIdentifier).Inc() } // IncDuplicateValidationErrorMetric increments the counter for the duplicate_validation_error_total metric. diff --git a/staging/src/k8s.io/apiserver/pkg/validation/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/validation/metrics_test.go index 5b038a5444b..016a6956eea 100644 --- a/staging/src/k8s.io/apiserver/pkg/validation/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/validation/metrics_test.go @@ -24,21 +24,28 @@ import ( "k8s.io/component-base/metrics/testutil" ) +const testIdentifier = "test_validation_identifier" +const anotherTestIdentifier = "another_test_validation_identifier" + // TestDeclarativeValidationMismatchMetric tests that the mismatch metric correctly increments once func TestDeclarativeValidationMismatchMetric(t *testing.T) { defer legacyregistry.Reset() defer ResetValidationMetricsInstance() // Increment the metric once - Metrics.IncDeclarativeValidationMismatchMetric() + Metrics.IncDeclarativeValidationMismatchMetric(testIdentifier) expected := ` + # HELP apiserver_validation_declarative_validation_parity_discrepancies_total [ALPHA] Number of discrepancies between declarative and handwritten validation, broken down by validation identifier. + # TYPE apiserver_validation_declarative_validation_parity_discrepancies_total counter + apiserver_validation_declarative_validation_parity_discrepancies_total{validation_identifier="test_validation_identifier"} 1 # HELP apiserver_validation_declarative_validation_mismatch_total [BETA] Number of times declarative validation results differed from handwritten validation results for core types. # TYPE apiserver_validation_declarative_validation_mismatch_total counter apiserver_validation_declarative_validation_mismatch_total 1 + ` - if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_mismatch_total"); err != nil { + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_parity_discrepancies_total", "apiserver_validation_declarative_validation_mismatch_total"); err != nil { t.Fatal(err) } } @@ -49,15 +56,19 @@ func TestDeclarativeValidationPanicMetric(t *testing.T) { defer ResetValidationMetricsInstance() // Increment the metric once - Metrics.IncDeclarativeValidationPanicMetric() + Metrics.IncDeclarativeValidationPanicMetric(testIdentifier) expected := ` + # HELP apiserver_validation_declarative_validation_panics_total [ALPHA] Number of panics in declarative validation, broken down by validation identifier. + # TYPE apiserver_validation_declarative_validation_panics_total counter + apiserver_validation_declarative_validation_panics_total{validation_identifier="test_validation_identifier"} 1 # HELP apiserver_validation_declarative_validation_panic_total [BETA] Number of times declarative validation has panicked during validation. # TYPE apiserver_validation_declarative_validation_panic_total counter apiserver_validation_declarative_validation_panic_total 1 ` - if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_panic_total"); err != nil { + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), + "apiserver_validation_declarative_validation_panic_total", "apiserver_validation_declarative_validation_panics_total"); err != nil { t.Fatal(err) } } @@ -87,17 +98,21 @@ func TestDeclarativeValidationMismatchMetricMultiple(t *testing.T) { defer ResetValidationMetricsInstance() // Increment the metric three times - Metrics.IncDeclarativeValidationMismatchMetric() - Metrics.IncDeclarativeValidationMismatchMetric() - Metrics.IncDeclarativeValidationMismatchMetric() + Metrics.IncDeclarativeValidationMismatchMetric(testIdentifier) + Metrics.IncDeclarativeValidationMismatchMetric(testIdentifier) + Metrics.IncDeclarativeValidationMismatchMetric(anotherTestIdentifier) expected := ` + # HELP apiserver_validation_declarative_validation_parity_discrepancies_total [ALPHA] Number of discrepancies between declarative and handwritten validation, broken down by validation identifier. + # TYPE apiserver_validation_declarative_validation_parity_discrepancies_total counter + apiserver_validation_declarative_validation_parity_discrepancies_total{validation_identifier="test_validation_identifier"} 2 + apiserver_validation_declarative_validation_parity_discrepancies_total{validation_identifier="another_test_validation_identifier"} 1 # HELP apiserver_validation_declarative_validation_mismatch_total [BETA] Number of times declarative validation results differed from handwritten validation results for core types. # TYPE apiserver_validation_declarative_validation_mismatch_total counter apiserver_validation_declarative_validation_mismatch_total 3 ` - if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_mismatch_total"); err != nil { + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_parity_discrepancies_total", "apiserver_validation_declarative_validation_mismatch_total"); err != nil { t.Fatal(err) } } @@ -108,17 +123,21 @@ func TestDeclarativeValidationPanicMetricMultiple(t *testing.T) { defer ResetValidationMetricsInstance() // Increment the metric three times - Metrics.IncDeclarativeValidationPanicMetric() - Metrics.IncDeclarativeValidationPanicMetric() - Metrics.IncDeclarativeValidationPanicMetric() + Metrics.IncDeclarativeValidationPanicMetric(testIdentifier) + Metrics.IncDeclarativeValidationPanicMetric(testIdentifier) + Metrics.IncDeclarativeValidationPanicMetric(anotherTestIdentifier) expected := ` + # HELP apiserver_validation_declarative_validation_panics_total [ALPHA] Number of panics in declarative validation, broken down by validation identifier. + # TYPE apiserver_validation_declarative_validation_panics_total counter + apiserver_validation_declarative_validation_panics_total{validation_identifier="test_validation_identifier"} 2 + apiserver_validation_declarative_validation_panics_total{validation_identifier="another_test_validation_identifier"} 1 # HELP apiserver_validation_declarative_validation_panic_total [BETA] Number of times declarative validation has panicked during validation. # TYPE apiserver_validation_declarative_validation_panic_total counter apiserver_validation_declarative_validation_panic_total 3 ` - if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_panic_total"); err != nil { + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_panic_total", "apiserver_validation_declarative_validation_panics_total"); err != nil { t.Fatal(err) } } @@ -150,8 +169,8 @@ func TestDeclarativeValidationMetricsReset(t *testing.T) { defer ResetValidationMetricsInstance() // Increment both metrics - Metrics.IncDeclarativeValidationMismatchMetric() - Metrics.IncDeclarativeValidationPanicMetric() + Metrics.IncDeclarativeValidationMismatchMetric(testIdentifier) + Metrics.IncDeclarativeValidationPanicMetric(testIdentifier) Metrics.IncDuplicateValidationErrorMetric() // Reset the metrics @@ -178,8 +197,8 @@ func TestDeclarativeValidationMetricsReset(t *testing.T) { } // Increment the metrics again to ensure they're still functional - Metrics.IncDeclarativeValidationMismatchMetric() - Metrics.IncDeclarativeValidationPanicMetric() + Metrics.IncDeclarativeValidationMismatchMetric(testIdentifier) + Metrics.IncDeclarativeValidationPanicMetric(testIdentifier) Metrics.IncDuplicateValidationErrorMetric() // Verify they've been incremented correctly @@ -193,11 +212,19 @@ func TestDeclarativeValidationMetricsReset(t *testing.T) { # HELP apiserver_validation_duplicate_validation_error_total [INTERNAL] Number of duplicate validation errors during validation. # TYPE apiserver_validation_duplicate_validation_error_total counter apiserver_validation_duplicate_validation_error_total 1 + # HELP apiserver_validation_declarative_validation_parity_discrepancies_total [ALPHA] Number of discrepancies between declarative and handwritten validation, broken down by validation identifier. + # TYPE apiserver_validation_declarative_validation_parity_discrepancies_total counter + apiserver_validation_declarative_validation_parity_discrepancies_total{validation_identifier="test_validation_identifier"} 1 + # HELP apiserver_validation_declarative_validation_panics_total [ALPHA] Number of panics in declarative validation, broken down by validation identifier. + # TYPE apiserver_validation_declarative_validation_panics_total counter + apiserver_validation_declarative_validation_panics_total{validation_identifier="test_validation_identifier"} 1 ` if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), "apiserver_validation_declarative_validation_mismatch_total", + "apiserver_validation_declarative_validation_parity_discrepancies_total", "apiserver_validation_declarative_validation_panic_total", + "apiserver_validation_declarative_validation_panics_total", "apiserver_validation_duplicate_validation_error_total"); err != nil { t.Fatal(err) }