diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index a5560d11fd9..4af2f1faf87 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -43,12 +43,13 @@ import ( "k8s.io/apiserver/pkg/apis/config/validation" "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server/healthz" + "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics" storagevalue "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope" envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2" kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2" - "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" + envelopemetrics "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" "k8s.io/apiserver/pkg/storage/value/encrypt/identity" "k8s.io/apiserver/pkg/storage/value/encrypt/secretbox" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -111,6 +112,9 @@ func init() { utilruntime.Must(apiserverconfig.AddToScheme(configScheme)) utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme)) codecs = serializer.NewCodecFactory(configScheme) + envelopemetrics.RegisterMetrics() + storagevalue.RegisterMetrics() + metrics.RegisterMetrics() } type kmsPluginHealthzResponse struct { @@ -133,6 +137,7 @@ type kmsv2PluginProbe struct { service kmsservice.Service lastResponse *kmsPluginHealthzResponse l *sync.Mutex + apiServerID string } type kmsHealthChecker []healthz.HealthChecker @@ -186,13 +191,13 @@ type EncryptionConfiguration struct { // It may launch multiple go routines whose lifecycle is controlled by ctx. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. // If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1. -func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*EncryptionConfiguration, error) { +func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) { config, contentHash, err := loadConfig(filepath, reload) if err != nil { return nil, fmt.Errorf("error while parsing file: %w", err) } - transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config) + transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID) if err != nil { return nil, fmt.Errorf("error while building transformers: %w", err) } @@ -217,9 +222,9 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E // getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config. // It may launch multiple go routines whose lifecycle is controlled by ctx. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. -func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) { +func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) { var kmsHealthChecks []healthz.HealthChecker - transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config) + transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID) if err != nil { return nil, nil, nil, err } @@ -238,7 +243,7 @@ type healthChecker interface { // getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config. // It may launch multiple go routines whose lifecycle is controlled by ctx. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. -func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) { +func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) { resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{} var probes []healthChecker var kmsUsed kmsState @@ -247,7 +252,7 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis for _, resourceConfig := range config.Resources { resourceConfig := resourceConfig - transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig) + transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID) if err != nil { return nil, nil, nil, err } @@ -454,10 +459,10 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C } if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil { - metrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode)) + envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode)) errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err)) } else { - metrics.RecordKeyIDFromStatus(h.name, response.KeyID) + envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID) // unconditionally append as we filter out nil errors below errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID()))) } @@ -499,7 +504,7 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig // prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config. // It may launch multiple go routines whose lifecycle is controlled by ctx. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. -func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) { +func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) { var transformers []storagevalue.PrefixTransformer var probes []healthChecker var kmsUsed kmsState @@ -527,7 +532,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox) case provider.KMS != nil: - transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS) + transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID) if transformerErr == nil { probes = append(probes, probe) kmsUsed.accumulate(used) @@ -686,7 +691,7 @@ func (s *kmsState) accumulate(other *kmsState) { // kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config. // It may launch multiple go routines whose lifecycle is controlled by ctx. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. -func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) { +func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) { kmsName := config.Name switch config.APIVersion { case kmsAPIVersionV1: @@ -732,14 +737,14 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig service: envelopeService, l: &sync.Mutex{}, lastResponse: &kmsPluginHealthzResponse{}, + apiServerID: apiServerID, } // initialize state so that Load always works probe.state.Store(&envelopekmsv2.State{}) primeAndProbeKMSv2(ctx, probe, kmsName) - transformer := storagevalue.PrefixTransformer{ - Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState), + Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID), Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"), } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index 704c10e5ed5..46c76284b73 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -207,7 +207,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 expectedKMSCloseGracePeriod := 46 * time.Second correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml" - identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false) + identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false, "") if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst) } @@ -218,7 +218,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 expectedKMSCloseGracePeriod = 32 * time.Second correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml" - aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false) + aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false, "") if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst) } @@ -227,7 +227,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { } invalidConfigWithAesGcm := "testdata/invalid-configs/invalid-aes-gcm.yaml" - _, err = LoadEncryptionConfig(ctx, invalidConfigWithAesGcm, false) + _, err = LoadEncryptionConfig(ctx, invalidConfigWithAesGcm, false, "") if !strings.Contains(errString(err), "error while parsing file") { t.Fatalf("should result in error while parsing configuration file: %s.\nThe file was:\n%s", err, invalidConfigWithAesGcm) } @@ -235,7 +235,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 expectedKMSCloseGracePeriod = 26 * time.Second correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml" - aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false) + aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false, "") if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst) } @@ -246,7 +246,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 expectedKMSCloseGracePeriod = 14 * time.Second correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml" - secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false) + secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false, "") if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst) } @@ -257,7 +257,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 expectedKMSCloseGracePeriod = 34 * time.Second correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml" - kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false) + kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false, "") if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst) } @@ -268,7 +268,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 expectedKMSCloseGracePeriod = 42 * time.Second correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml" - kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false) + kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false, "") if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First) } @@ -343,7 +343,7 @@ func TestKMSv1Deprecation(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, testCase.kmsv1Enabled)() kmsv1Config := "testdata/valid-configs/kms/multiple-providers.yaml" - _, err := LoadEncryptionConfig(testContext(t), kmsv1Config, false) + _, err := LoadEncryptionConfig(testContext(t), kmsv1Config, false, "") if !strings.Contains(errString(err), testCase.expectedErr) { t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err)) } @@ -384,7 +384,7 @@ func TestKMSvsEnablement(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, testCase.kmsv2Enabled)() - _, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false) + _, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false, "") if !strings.Contains(errString(err), testCase.expectedErr) { t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err)) @@ -486,7 +486,7 @@ func TestKMSvsEnablement(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // cancel this upfront so the kms v2 checks do not block - _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &tt.config) + _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &tt.config, "") if err == nil { if kmsUsed == nil || kmsUsed.v2Used != tt.wantV2Used { t.Fatalf("unexpected kmsUsed value, expected: %v, got: %v", tt.wantV2Used, kmsUsed) @@ -729,7 +729,7 @@ func TestKMSMaxTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // cancel this upfront so the kms v2 checks do not block - _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &testCase.config) + _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &testCase.config, "") if !strings.Contains(errString(err), testCase.expectedErr) { t.Fatalf("expecting error calling prefixTransformersAndProbes, expected: %s, got: %s", testCase.expectedErr, errString(err)) @@ -752,8 +752,9 @@ func TestKMSPluginHealthz(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() kmsv2Probe := &kmsv2PluginProbe{ - name: "foo", - ttl: 3 * time.Second, + name: "foo", + ttl: 3 * time.Second, + apiServerID: "", } keyID := "1" kmsv2Probe.state.Store(&envelopekmsv2.State{EncryptedObject: kmstypes.EncryptedObject{KeyID: keyID}}) @@ -853,7 +854,7 @@ func TestKMSPluginHealthz(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // cancel this upfront so the kms v2 healthz check poll does not run - _, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config) + _, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, "") if err != nil { t.Fatal(err) } @@ -1298,7 +1299,7 @@ func TestWildcardMasking(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - _, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config) + _, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "") if errString(err) != tc.expectedError { t.Errorf("expected error %s but got %s", tc.expectedError, errString(err)) } @@ -1453,7 +1454,7 @@ func TestWildcardStructure(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config) + transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "") if errString(err) != tc.errorValue { t.Errorf("expected error %s but got %s", tc.errorValue, errString(err)) } @@ -1724,7 +1725,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin ctx := testContext(t) t.Helper() - encryptionConfiguration, err := LoadEncryptionConfig(ctx, encryptionConfigPath, false) + encryptionConfiguration, err := LoadEncryptionConfig(ctx, encryptionConfigPath, false, "") if err != nil { t.Fatal(err) } @@ -2123,6 +2124,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called "panda", h.getCurrentState, + "", ) dataCtx := value.DefaultContext(sampleContextText) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go index 94782ccbacd..5bb90995b9e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go @@ -50,6 +50,13 @@ type DynamicKMSEncryptionConfigContent struct { // dynamicTransformers updates the transformers when encryption config file changes. dynamicTransformers *encryptionconfig.DynamicTransformers + + // identity of the api server + apiServerID string +} + +func init() { + metrics.RegisterMetrics() } // NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file. @@ -57,6 +64,7 @@ func NewDynamicEncryptionConfiguration( name, filePath string, dynamicTransformers *encryptionconfig.DynamicTransformers, configContentHash string, + apiServerID string, ) *DynamicKMSEncryptionConfigContent { encryptionConfig := &DynamicKMSEncryptionConfigContent{ name: name, @@ -64,6 +72,7 @@ func NewDynamicEncryptionConfiguration( lastLoadedEncryptionConfigHash: configContentHash, dynamicTransformers: dynamicTransformers, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name), + apiServerID: apiServerID, } encryptionConfig.queue.Add(workqueueKey) // to avoid missing any file changes that occur in between the initial load and Run @@ -172,11 +181,11 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex } if updatedEffectiveConfig && err == nil { - metrics.RecordEncryptionConfigAutomaticReloadSuccess() + metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID) } if err != nil { - metrics.RecordEncryptionConfigAutomaticReloadFailure() + metrics.RecordEncryptionConfigAutomaticReloadFailure(d.apiServerID) utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err)) // add dummy item back to the queue to trigger file content processing. d.queue.AddRateLimited(key) @@ -224,7 +233,7 @@ func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context. err error, ) { // this code path will only execute if reload=true. So passing true explicitly. - encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true) + encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true, d.apiServerID) if err != nil { return nil, false, err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go index 196ef372bc3..c2a007c408a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go @@ -46,6 +46,7 @@ func TestProcessEncryptionConfig(t *testing.T) { testCase.filePath, nil, "", + "", ) _, _, err := d.processEncryptionConfig(ctx) @@ -110,6 +111,7 @@ func TestWatchEncryptionConfigFile(t *testing.T) { testFilePath, nil, "", + "", ) errs := make(chan error, 1) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go index 799b584cf7a..70414035fed 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go @@ -17,6 +17,9 @@ limitations under the License. package metrics import ( + "crypto/sha256" + "fmt" + "hash" "sync" "k8s.io/component-base/metrics" @@ -29,24 +32,26 @@ const ( ) var ( - encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter( + encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec( &metrics.CounterOpts{ Namespace: namespace, Subsystem: subsystem, Name: "automatic_reload_failures_total", - Help: "Total number of failed automatic reloads of encryption configuration.", + Help: "Total number of failed automatic reloads of encryption configuration split by apiserver identity.", StabilityLevel: metrics.ALPHA, }, + []string{"apiserver_id_hash"}, ) - encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter( + encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec( &metrics.CounterOpts{ Namespace: namespace, Subsystem: subsystem, Name: "automatic_reload_success_total", - Help: "Total number of successful automatic reloads of encryption configuration.", + Help: "Total number of successful automatic reloads of encryption configuration split by apiserver identity.", StabilityLevel: metrics.ALPHA, }, + []string{"apiserver_id_hash"}, ) encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec( @@ -54,33 +59,53 @@ var ( Namespace: namespace, Subsystem: subsystem, Name: "automatic_reload_last_timestamp_seconds", - Help: "Timestamp of the last successful or failed automatic reload of encryption configuration.", + Help: "Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.", StabilityLevel: metrics.ALPHA, }, - []string{"status"}, + []string{"status", "apiserver_id_hash"}, ) ) var registerMetrics sync.Once +var hashPool *sync.Pool func RegisterMetrics() { registerMetrics.Do(func() { + hashPool = &sync.Pool{ + New: func() interface{} { + return sha256.New() + }, + } legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal) legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal) legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds) }) } -func RecordEncryptionConfigAutomaticReloadFailure() { - encryptionConfigAutomaticReloadFailureTotal.Inc() - recordEncryptionConfigAutomaticReloadTimestamp("failure") +func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) { + apiServerIDHash := getHash(apiServerID) + encryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc() + recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash) } -func RecordEncryptionConfigAutomaticReloadSuccess() { - encryptionConfigAutomaticReloadSuccessTotal.Inc() - recordEncryptionConfigAutomaticReloadTimestamp("success") +func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) { + apiServerIDHash := getHash(apiServerID) + encryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc() + recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash) } -func recordEncryptionConfigAutomaticReloadTimestamp(result string) { - encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime() +func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) { + encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime() +} + +func getHash(data string) string { + if len(data) == 0 { + return "" + } + h := hashPool.Get().(hash.Hash) + h.Reset() + h.Write([]byte(data)) + dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil)) + hashPool.Put(h) + return dataHash } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go index d6b34faeedd..fdf89e9bf95 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go @@ -24,11 +24,16 @@ import ( "k8s.io/component-base/metrics/testutil" ) +const ( + testAPIServerID = "testAPIServerID" + testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" +) + func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) { expectedValue := ` - # HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration. + # HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter - apiserver_encryption_config_controller_automatic_reload_failures_total 1 + apiserver_encryption_config_controller_automatic_reload_failures_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"} 1 ` metrics := []string{ namespace + "_" + subsystem + "_automatic_reload_failures_total", @@ -37,7 +42,7 @@ func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) { encryptionConfigAutomaticReloadFailureTotal.Reset() RegisterMetrics() - RecordEncryptionConfigAutomaticReloadFailure() + RecordEncryptionConfigAutomaticReloadFailure(testAPIServerID) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { t.Fatal(err) } @@ -45,9 +50,9 @@ func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) { func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) { expectedValue := ` - # HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration. + # HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter - apiserver_encryption_config_controller_automatic_reload_success_total 1 + apiserver_encryption_config_controller_automatic_reload_success_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"} 1 ` metrics := []string{ namespace + "_" + subsystem + "_automatic_reload_success_total", @@ -56,7 +61,7 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) { encryptionConfigAutomaticReloadSuccessTotal.Reset() RegisterMetrics() - RecordEncryptionConfigAutomaticReloadSuccess() + RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { t.Fatal(err) } @@ -70,18 +75,18 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { }{ { expectedValue: ` - # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration. + # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge - apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="failure"} 1.689101941e+09 + apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="failure"} 1.689101941e+09 `, resultLabel: "failure", timestamp: 1689101941, }, { expectedValue: ` - # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration. + # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge - apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="success"} 1.689101941e+09 + apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1.689101941e+09 `, resultLabel: "success", timestamp: 1689101941, @@ -95,7 +100,7 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { for _, tc := range testCases { encryptionConfigAutomaticReloadLastTimestampSeconds.Reset() - encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel).Set(float64(tc.timestamp)) + encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel, testAPIServerIDHash).Set(float64(tc.timestamp)) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 57e9c1a9f13..2d8741ebd1f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -294,7 +294,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro } }() - encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload) + encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID) if err != nil { return err } @@ -318,6 +318,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro s.EncryptionProviderConfigFilepath, dynamicTransformers, encryptionConfiguration.EncryptionFileContentHash, + c.APIServerID, ) go dynamicEncryptionConfigController.Run(ctxServer) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go index bdf5a10d942..23de3717a18 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go @@ -110,22 +110,24 @@ type envelopeTransformer struct { stateFunc StateFunc // cache is a thread-safe expiring lru cache which caches decrypted DEKs indexed by their encrypted form. - cache *simpleCache + cache *simpleCache + apiServerID string } // NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme. // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to // the data items they encrypt. -func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc) value.Transformer { - return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, cacheTTL, clock.RealClock{}) +func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string) value.Transformer { + return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, apiServerID, cacheTTL, clock.RealClock{}) } -func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, cacheTTL time.Duration, clock clock.Clock) value.Transformer { +func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string, cacheTTL time.Duration, clock clock.Clock) value.Transformer { return &envelopeTransformer{ envelopeService: envelopeService, providerName: providerName, stateFunc: stateFunc, cache: newSimpleCache(clock, cacheTTL, providerName), + apiServerID: apiServerID, } } @@ -178,7 +180,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b return nil, false, err } } - metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID) + metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID, t.apiServerID) out, stale, err := transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx) if err != nil { @@ -218,7 +220,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt return nil, err } - metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID) + metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID, t.apiServerID) encObjectCopy := state.EncryptedObject encObjectCopy.EncryptedData = result diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go index dbe489e7749..f52d4cd9d46 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go @@ -49,10 +49,12 @@ import ( ) const ( - testText = "abcdefghijklmnopqrstuvwxyz" - testContextText = "0123456789" - testKeyHash = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" - testKeyVersion = "1" + testText = "abcdefghijklmnopqrstuvwxyz" + testContextText = "0123456789" + testKeyHash = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" + testKeyVersion = "1" + testAPIServerID = "testAPIServerID" + testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" ) // testEnvelopeService is a mock Envelope service which can be used to simulate remote Envelope services @@ -178,7 +180,7 @@ func TestEnvelopeCaching(t *testing.T) { } transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, - func() (State, error) { return state, nil }, + func() (State, error) { return state, nil }, testAPIServerID, tt.cacheTTL, fakeClock) dataCtx := value.DefaultContext(testContextText) @@ -319,7 +321,7 @@ func TestEnvelopeTransformerStaleness(t *testing.T) { var stateErr error transformer := NewEnvelopeTransformer(envelopeService, testProviderName, - func() (State, error) { return state, stateErr }, + func() (State, error) { return state, stateErr }, testAPIServerID, ) dataCtx := value.DefaultContext(testContextText) @@ -376,7 +378,7 @@ func TestEnvelopeTransformerStateFunc(t *testing.T) { stateErr := fmt.Errorf("some state error") transformer := NewEnvelopeTransformer(envelopeService, testProviderName, - func() (State, error) { return state, stateErr }, + func() (State, error) { return state, stateErr }, testAPIServerID, ) dataCtx := value.DefaultContext(testContextText) @@ -513,6 +515,7 @@ func TestTransformToStorageError(t *testing.T) { envelopeService.SetAnnotations(tt.annotations) transformer := NewEnvelopeTransformer(envelopeService, testProviderName, testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool()), + testAPIServerID, ) dataCtx := value.DefaultContext(testContextText) @@ -838,6 +841,7 @@ func TestEnvelopeMetrics(t *testing.T) { envelopeService := newTestEnvelopeService() transformer := NewEnvelopeTransformer(envelopeService, testProviderName, testStateFunc(testContext(t), envelopeService, clock.RealClock{}, randomBool()), + testAPIServerID, ) dataCtx := value.DefaultContext(testContextText) @@ -859,11 +863,11 @@ func TestEnvelopeMetrics(t *testing.T) { "apiserver_envelope_encryption_key_id_hash_total", }, want: fmt.Sprintf(` - # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. + # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity. # TYPE apiserver_envelope_encryption_key_id_hash_total counter - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - `, testKeyHash, testProviderName, metrics.FromStorageLabel, testKeyHash, testProviderName, metrics.ToStorageLabel), + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + `, testAPIServerIDHash, testKeyHash, testProviderName, metrics.FromStorageLabel, testAPIServerIDHash, testKeyHash, testProviderName, metrics.ToStorageLabel), }, } @@ -931,10 +935,10 @@ func TestEnvelopeMetricsCache(t *testing.T) { transformer1 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { // return different states to ensure we get expected number of cache keys after restart on decryption return testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool())() - }) - transformer2 := NewEnvelopeTransformer(envelopeService, provider2, func() (State, error) { return state, nil }) + }, testAPIServerID) + transformer2 := NewEnvelopeTransformer(envelopeService, provider2, func() (State, error) { return state, nil }, testAPIServerID) // used for restart - transformer3 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { return state, nil }) + transformer3 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { return state, nil }, testAPIServerID) var transformedDatas [][]byte for j := 0; j < numOfStates; j++ { transformedData, err := transformer1.TransformToStorage(ctx, []byte(testText), dataCtx) @@ -1029,8 +1033,7 @@ func TestEnvelopeLogging(t *testing.T) { envelopeService := newTestEnvelopeService() fakeClock := testingclock.NewFakeClock(time.Now()) transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, - testStateFunc(tc.ctx, envelopeService, clock.RealClock{}, randomBool()), - 1*time.Second, fakeClock) + testStateFunc(tc.ctx, envelopeService, clock.RealClock{}, randomBool()), testAPIServerID, 1*time.Second, fakeClock) dataCtx := value.DefaultContext([]byte(testContextText)) originalText := []byte(testText) @@ -1082,7 +1085,7 @@ func TestCacheNotCorrupted(t *testing.T) { } transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, - func() (State, error) { return state, nil }, + func() (State, error) { return state, nil }, testAPIServerID, 1*time.Second, fakeClock) dataCtx := value.DefaultContext(testContextText) @@ -1108,7 +1111,7 @@ func TestCacheNotCorrupted(t *testing.T) { } transformer = newEnvelopeTransformerWithClock(envelopeService, testProviderName, - func() (State, error) { return state, nil }, + func() (State, error) { return state, nil }, testAPIServerID, 1*time.Second, fakeClock) transformedData2, err := transformer.TransformToStorage(ctx, originalText, dataCtx) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go index be0f6dde932..63723648c97 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go @@ -44,6 +44,7 @@ type metricLabels struct { transformationType string providerName string keyIDHash string + apiServerIDHash string } /* @@ -107,21 +108,21 @@ var ( // keyIDHashTotal is the number of times a keyID is used // e.g. apiserver_envelope_encryption_key_id_hash_total counter - // apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256", + // apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256",key_id_hash="sha256", // provider_name="providerName",transformation_type="from_storage"} 1 KeyIDHashTotal = metrics.NewCounterVec( &metrics.CounterOpts{ Namespace: namespace, Subsystem: subsystem, Name: "key_id_hash_total", - Help: "Number of times a keyID is used split by transformation type and provider.", + Help: "Number of times a keyID is used split by transformation type, provider, and apiserver identity.", StabilityLevel: metrics.ALPHA, }, - []string{"transformation_type", "provider_name", "key_id_hash"}, + []string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"}, ) // keyIDHashLastTimestampSeconds is the last time in seconds when a keyID was used - // e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09 + // e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09 KeyIDHashLastTimestampSeconds = metrics.NewGaugeVec( &metrics.GaugeOpts{ Namespace: namespace, @@ -130,11 +131,11 @@ var ( Help: "The last time in seconds when a keyID was used.", StabilityLevel: metrics.ALPHA, }, - []string{"transformation_type", "provider_name", "key_id_hash"}, + []string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"}, ) // keyIDHashStatusLastTimestampSeconds is the last time in seconds when a keyID was returned by the Status RPC call. - // e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09 + // e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09 KeyIDHashStatusLastTimestampSeconds = metrics.NewGaugeVec( &metrics.GaugeOpts{ Namespace: namespace, @@ -143,7 +144,7 @@ var ( Help: "The last time in seconds when a keyID was returned by the Status RPC call.", StabilityLevel: metrics.ALPHA, }, - []string{"provider_name", "key_id_hash"}, + []string{"provider_name", "key_id_hash", "apiserver_id_hash"}, ) InvalidKeyIDFromStatusTotal = metrics.NewCounterVec( @@ -182,19 +183,19 @@ func registerLRUMetrics() { keyIDHashTotalMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) { item := key.(metricLabels) - if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted { + if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted { klog.InfoS("Deleted keyIDHashTotalMetricLabels", "transformationType", item.transformationType, - "providerName", item.providerName, "keyIDHash", item.keyIDHash) + "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash) } - if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted { + if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted { klog.InfoS("Deleted keyIDHashLastTimestampSecondsMetricLabels", "transformationType", item.transformationType, - "providerName", item.providerName, "keyIDHash", item.keyIDHash) + "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash) } }) keyIDHashStatusLastTimestampSecondsMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) { item := key.(metricLabels) - if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash); deleted { - klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash) + if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash, item.apiServerIDHash); deleted { + klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash) } }) } @@ -218,22 +219,22 @@ func RegisterMetrics() { } // RecordKeyID records total count and last time in seconds when a KeyID was used for TransformFromStorage and TransformToStorage operations -func RecordKeyID(transformationType, providerName, keyID string) { +func RecordKeyID(transformationType, providerName, keyID, apiServerID string) { lockRecordKeyID.Lock() defer lockRecordKeyID.Unlock() - keyIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID) - KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash).Inc() - KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash).SetToCurrentTime() + keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID, apiServerID) + KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).Inc() + KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).SetToCurrentTime() } // RecordKeyIDFromStatus records last time in seconds when a KeyID was returned by the Status RPC call. -func RecordKeyIDFromStatus(providerName, keyID string) { +func RecordKeyIDFromStatus(providerName, keyID, apiServerID string) { lockRecordKeyIDStatus.Lock() defer lockRecordKeyIDStatus.Unlock() - keyIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID) - KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash).SetToCurrentTime() + keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID, apiServerID) + KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash, apiServerIDHash).SetToCurrentTime() } func RecordInvalidKeyIDFromStatus(providerName, errCode string) { @@ -297,24 +298,25 @@ func getErrorCode(err error) string { } func getHash(data string) string { + if len(data) == 0 { + return "" + } h := hashPool.Get().(hash.Hash) h.Reset() h.Write([]byte(data)) - result := fmt.Sprintf("sha256:%x", h.Sum(nil)) + dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil)) hashPool.Put(h) - return result + return dataHash } -func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID string) string { - keyIDHash := "" - // only get hash if the keyID is not empty - if len(keyID) > 0 { - keyIDHash = getHash(keyID) - } +func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID, apiServerID string) (string, string) { + keyIDHash := getHash(keyID) + apiServerIDHash := getHash(apiServerID) c.Add(metricLabels{ transformationType: transformationType, providerName: providerName, keyIDHash: keyIDHash, + apiServerIDHash: apiServerIDHash, }, nil) // value is irrelevant, this is a set and not a map - return keyIDHash + return keyIDHash, apiServerIDHash } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go index b72099c77b0..e241a3a4ddd 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go @@ -36,6 +36,8 @@ const ( testKeyHash2 = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" testKeyHash3 = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" testProviderNameForMetric = "providerName" + testAPIServerID = "testAPIServerID" + testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" ) var ( @@ -204,6 +206,7 @@ func TestRecordKeyID_Serial(t *testing.T) { metrics []string providerName string transformationType string + apiServerID string want string }{ { @@ -214,11 +217,12 @@ func TestRecordKeyID_Serial(t *testing.T) { }, providerName: testProviderNameForMetric, transformationType: FromStorageLabel, + apiServerID: testAPIServerID, want: fmt.Sprintf(` - # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. + # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity. # TYPE apiserver_envelope_encryption_key_id_hash_total counter - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - `, testKeyHash1, testProviderNameForMetric, FromStorageLabel), + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + `, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel), }, { desc: "keyIDHash total more labels", @@ -228,12 +232,13 @@ func TestRecordKeyID_Serial(t *testing.T) { }, providerName: testProviderNameForMetric, transformationType: FromStorageLabel, + apiServerID: testAPIServerID, want: fmt.Sprintf(` - # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. + # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity. # TYPE apiserver_envelope_encryption_key_id_hash_total counter - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - `, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testKeyHash2, testProviderNameForMetric, FromStorageLabel), + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + `, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel), }, { desc: "keyIDHash total same labels", @@ -243,12 +248,13 @@ func TestRecordKeyID_Serial(t *testing.T) { }, providerName: testProviderNameForMetric, transformationType: FromStorageLabel, + apiServerID: testAPIServerID, want: fmt.Sprintf(` - # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. + # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity. # TYPE apiserver_envelope_encryption_key_id_hash_total counter - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2 - `, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testKeyHash2, testProviderNameForMetric, FromStorageLabel), + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2 + `, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel), }, { desc: "keyIDHash total exceeds limit, remove first label, and empty keyID", @@ -258,12 +264,29 @@ func TestRecordKeyID_Serial(t *testing.T) { }, providerName: testProviderNameForMetric, transformationType: FromStorageLabel, + apiServerID: testAPIServerID, want: fmt.Sprintf(` - # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. + # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity. # TYPE apiserver_envelope_encryption_key_id_hash_total counter - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2 - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - `, testKeyHash2, testProviderNameForMetric, FromStorageLabel, "", testProviderNameForMetric, FromStorageLabel), + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2 + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + `, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, "", testProviderNameForMetric, FromStorageLabel), + }, + { + desc: "keyIDHash total exceeds limit, remove first label, empty keyID, and empty testAPIServerID", + keyID: "", + metrics: []string{ + "apiserver_envelope_encryption_key_id_hash_total", + }, + providerName: testProviderNameForMetric, + transformationType: FromStorageLabel, + apiServerID: "", + want: fmt.Sprintf(` + # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity. + # TYPE apiserver_envelope_encryption_key_id_hash_total counter + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + `, testAPIServerIDHash, "", testProviderNameForMetric, FromStorageLabel, "", "", testProviderNameForMetric, FromStorageLabel), }, { desc: "keyIDHash total exceeds limit 2, remove first label", @@ -273,12 +296,13 @@ func TestRecordKeyID_Serial(t *testing.T) { }, providerName: testProviderNameForMetric, transformationType: FromStorageLabel, + apiServerID: "", want: fmt.Sprintf(` - # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. + # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity. # TYPE apiserver_envelope_encryption_key_id_hash_total counter - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 - `, "", testProviderNameForMetric, FromStorageLabel, testKeyHash1, testProviderNameForMetric, FromStorageLabel), + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 + `, "", "", testProviderNameForMetric, FromStorageLabel, "", testKeyHash1, testProviderNameForMetric, FromStorageLabel), }, } @@ -289,7 +313,7 @@ func TestRecordKeyID_Serial(t *testing.T) { for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { - RecordKeyID(tt.transformationType, tt.providerName, tt.keyID) + RecordKeyID(tt.transformationType, tt.providerName, tt.keyID, tt.apiServerID) // We are not resetting the metric here as each test is not independent in order to validate the behavior // when the metric labels exceed the limit to ensure the labels are not unbounded. if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { @@ -313,12 +337,14 @@ func TestRecordKeyIDLRUKey(t *testing.T) { go func() { defer wg.Done() keyID := rand.String(32) + apiServerID := rand.String(32) key := metricLabels{ transformationType: rand.String(32), providerName: rand.String(32), keyIDHash: getHash(keyID), + apiServerIDHash: getHash(apiServerID), } - RecordKeyID(key.transformationType, key.providerName, keyID) + RecordKeyID(key.transformationType, key.providerName, keyID, apiServerID) }() } wg.Wait() @@ -359,11 +385,13 @@ func TestRecordKeyIDFromStatus(t *testing.T) { go func() { defer wg.Done() keyID := rand.String(32) + apiServerID := rand.String(32) key := metricLabels{ - providerName: rand.String(32), - keyIDHash: getHash(keyID), + providerName: rand.String(32), + keyIDHash: getHash(keyID), + apiServerIDHash: getHash(apiServerID), } - RecordKeyIDFromStatus(key.providerName, keyID) + RecordKeyIDFromStatus(key.providerName, keyID, apiServerID) }() } wg.Wait() diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 9d659205cf8..6985f7f563e 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -29,21 +29,27 @@ import ( "math/rand" "os" "path/filepath" + "regexp" "strings" "testing" "time" + "github.com/google/go-cmp/cmp" clientv3 "go.etcd.io/etcd/client/v3" "golang.org/x/crypto/cryptobyte" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" + genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1" "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" featuregatetesting "k8s.io/component-base/featuregate/testing" kmsapi "k8s.io/kms/apis/v1beta1" "k8s.io/kubernetes/test/integration" @@ -308,6 +314,8 @@ resources: cachesize: 1000 endpoint: unix:///@kms-provider.sock ` + + genericapiserver.SetHostnameFuncForTests("testAPIServerID") _ = mock.NewBase64Plugin(t, "@kms-provider.sock") var restarted bool test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig) @@ -319,6 +327,26 @@ resources: test.cleanUp() } }() + ctx := testContext(t) + + // the global metrics registry persists across test runs - reset it here so we can make assertions + copyConfig := rest.CopyConfig(test.kubeAPIServer.ClientConfig) + copyConfig.GroupVersion = &schema.GroupVersion{} + copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer() + rc, err := rest.RESTClientFor(copyConfig) + if err != nil { + t.Fatal(err) + } + if err := rc.Delete().AbsPath("/metrics").Do(ctx).Error(); err != nil { + t.Fatal(err) + } + + // assert that the metrics we collect during the test run match expectations + // NOTE: 2 successful automatic reload resulted from 2 config file updates + wantMetricStrings := []string{ + `apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, + `apiserver_encryption_config_controller_automatic_reload_success_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795"} 2`, + } test.secret, err = test.createSecret(testSecret, testNamespace) if err != nil { @@ -378,7 +406,7 @@ resources: // run storage migration // get secrets - ctx := testContext(t) + secretsList, err := test.restClient.CoreV1().Secrets("").List( ctx, metav1.ListOptions{}, @@ -522,6 +550,34 @@ resources: if _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}); err != nil { t.Fatalf("failed to list configmaps, err: %v", err) } + + // recreate rest client with the new transformTest + copyConfig = rest.CopyConfig(test.kubeAPIServer.ClientConfig) + copyConfig.GroupVersion = &schema.GroupVersion{} + copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer() + rc, err = rest.RESTClientFor(copyConfig) + if err != nil { + t.Fatal(err) + } + defer func() { + body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx) + if err != nil { + t.Fatal(err) + } + var gotMetricStrings []string + trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`) + for _, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(line, "apiserver_encryption_config_controller_") { + if strings.Contains(line, "_seconds") { + line = trimFP.ReplaceAllString(line, `$1`) + "} FP" // ignore floating point metric values + } + gotMetricStrings = append(gotMetricStrings, line) + } + } + if diff := cmp.Diff(wantMetricStrings, gotMetricStrings); diff != "" { + t.Errorf("unexpected metrics diff (-want +got): %s", diff) + } + }() } func TestEncryptAll(t *testing.T) { diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 5c10e85025b..2ad70311baf 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -199,7 +199,7 @@ resources: name: kms-provider endpoint: unix:///@kms-provider.sock ` - + genericapiserver.SetHostnameFuncForTests("testAPIServerID") providerName := "kms-provider" pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") @@ -226,10 +226,10 @@ resources: // assert that the metrics we collect during the test run match expectations wantMetricStrings := []string{ `apiserver_envelope_encryption_dek_source_cache_size{provider_name="kms-provider"} 1`, - `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`, - `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`, - `apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`, - `apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`, + `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`, + `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`, + `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`, + `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`, } defer func() { body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)