github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-06 02:34:03 +00:00
Code Issues Projects Releases Wiki Activity

kmsv2: add apiserver identity to metrics

Browse Source 
Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>
This commit is contained in:
Rita Zhang 2023-09-05 13:03:18 -07:00
parent 37cf2638c9
commit 43ccf6c4e8
No known key found for this signature in database
GPG Key ID: 0B1D9C98A2BFE852
13 changed files with 283 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go
View File

@ -43,12 +43,13 @@ import (
"k8s.io/apiserver/pkg/apis/config/validation" "k8s.io/apiserver/pkg/apis/config/validation"
"k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
storagevalue "k8s.io/apiserver/pkg/storage/value" storagevalue "k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2" envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2" kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" envelopemetrics "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
"k8s.io/apiserver/pkg/storage/value/encrypt/identity" "k8s.io/apiserver/pkg/storage/value/encrypt/identity"
"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox" "k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
@ -111,6 +112,9 @@ func init() {
utilruntime.Must(apiserverconfig.AddToScheme(configScheme)) utilruntime.Must(apiserverconfig.AddToScheme(configScheme))
utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme)) utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme))
codecs = serializer.NewCodecFactory(configScheme) codecs = serializer.NewCodecFactory(configScheme)
envelopemetrics.RegisterMetrics()
storagevalue.RegisterMetrics()
metrics.RegisterMetrics()
} }
type kmsPluginHealthzResponse struct { type kmsPluginHealthzResponse struct {
@ -133,6 +137,7 @@ type kmsv2PluginProbe struct {
service kmsservice.Service service kmsservice.Service
lastResponse *kmsPluginHealthzResponse lastResponse *kmsPluginHealthzResponse
l *sync.Mutex l *sync.Mutex
apiServerID string
} }
type kmsHealthChecker []healthz.HealthChecker type kmsHealthChecker []healthz.HealthChecker
@ -186,13 +191,13 @@ type EncryptionConfiguration struct {
// It may launch multiple go routines whose lifecycle is controlled by ctx. // It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1. // If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*EncryptionConfiguration, error) { func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) {
config, contentHash, err := loadConfig(filepath, reload) config, contentHash, err := loadConfig(filepath, reload)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while parsing file: %w", err) return nil, fmt.Errorf("error while parsing file: %w", err)
} }
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config) transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while building transformers: %w", err) return nil, fmt.Errorf("error while building transformers: %w", err)
} }
@ -217,9 +222,9 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E
// getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config. // getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
// It may launch multiple go routines whose lifecycle is controlled by ctx. // It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) { func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
var kmsHealthChecks []healthz.HealthChecker var kmsHealthChecks []healthz.HealthChecker
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config) transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
@ -238,7 +243,7 @@ type healthChecker interface {
// getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config. // getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
// It may launch multiple go routines whose lifecycle is controlled by ctx. // It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) { func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{} resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
var probes []healthChecker var probes []healthChecker
var kmsUsed kmsState var kmsUsed kmsState
@ -247,7 +252,7 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
for _, resourceConfig := range config.Resources { for _, resourceConfig := range config.Resources {
resourceConfig := resourceConfig resourceConfig := resourceConfig
transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig) transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
@ -454,10 +459,10 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
} }
if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil { if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
metrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode)) envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err)) errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
} else { } else {
metrics.RecordKeyIDFromStatus(h.name, response.KeyID) envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID)
// unconditionally append as we filter out nil errors below // unconditionally append as we filter out nil errors below
errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID()))) errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID())))
} }
@ -499,7 +504,7 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
// prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config. // prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
// It may launch multiple go routines whose lifecycle is controlled by ctx. // It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) { func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
var transformers []storagevalue.PrefixTransformer var transformers []storagevalue.PrefixTransformer
var probes []healthChecker var probes []healthChecker
var kmsUsed kmsState var kmsUsed kmsState
@ -527,7 +532,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox) transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
case provider.KMS != nil: case provider.KMS != nil:
transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS) transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID)
if transformerErr == nil { if transformerErr == nil {
probes = append(probes, probe) probes = append(probes, probe)
kmsUsed.accumulate(used) kmsUsed.accumulate(used)
@ -686,7 +691,7 @@ func (s *kmsState) accumulate(other *kmsState) {
// kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config. // kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
// It may launch multiple go routines whose lifecycle is controlled by ctx. // It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched. // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) { func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
kmsName := config.Name kmsName := config.Name
switch config.APIVersion { switch config.APIVersion {
case kmsAPIVersionV1: case kmsAPIVersionV1:
@ -732,14 +737,14 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
service: envelopeService, service: envelopeService,
l: &sync.Mutex{}, l: &sync.Mutex{},
lastResponse: &kmsPluginHealthzResponse{}, lastResponse: &kmsPluginHealthzResponse{},
apiServerID: apiServerID,
} }
// initialize state so that Load always works // initialize state so that Load always works
probe.state.Store(&envelopekmsv2.State{}) probe.state.Store(&envelopekmsv2.State{})
primeAndProbeKMSv2(ctx, probe, kmsName) primeAndProbeKMSv2(ctx, probe, kmsName)
transformer := storagevalue.PrefixTransformer{ transformer := storagevalue.PrefixTransformer{
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState), Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID),
Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"), Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"),
} }

32
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go
View File

@ -207,7 +207,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
expectedKMSCloseGracePeriod := 46 * time.Second expectedKMSCloseGracePeriod := 46 * time.Second
correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml" correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml"
identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false) identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false, "")
if err != nil { if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst) t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst)
} }
@ -218,7 +218,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
expectedKMSCloseGracePeriod = 32 * time.Second expectedKMSCloseGracePeriod = 32 * time.Second
correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml" correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml"
aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false) aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false, "")
if err != nil { if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst) t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst)
} }
@ -227,7 +227,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
} }
invalidConfigWithAesGcm := "testdata/invalid-configs/invalid-aes-gcm.yaml" invalidConfigWithAesGcm := "testdata/invalid-configs/invalid-aes-gcm.yaml"
_, err = LoadEncryptionConfig(ctx, invalidConfigWithAesGcm, false) _, err = LoadEncryptionConfig(ctx, invalidConfigWithAesGcm, false, "")
if !strings.Contains(errString(err), "error while parsing file") { if !strings.Contains(errString(err), "error while parsing file") {
t.Fatalf("should result in error while parsing configuration file: %s.\nThe file was:\n%s", err, invalidConfigWithAesGcm) t.Fatalf("should result in error while parsing configuration file: %s.\nThe file was:\n%s", err, invalidConfigWithAesGcm)
} }
@ -235,7 +235,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
expectedKMSCloseGracePeriod = 26 * time.Second expectedKMSCloseGracePeriod = 26 * time.Second
correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml" correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml"
aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false) aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false, "")
if err != nil { if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst) t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst)
} }
@ -246,7 +246,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
expectedKMSCloseGracePeriod = 14 * time.Second expectedKMSCloseGracePeriod = 14 * time.Second
correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml" correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml"
secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false) secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false, "")
if err != nil { if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst) t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst)
} }
@ -257,7 +257,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
expectedKMSCloseGracePeriod = 34 * time.Second expectedKMSCloseGracePeriod = 34 * time.Second
correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml" correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml"
kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false) kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false, "")
if err != nil { if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst) t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst)
} }
@ -268,7 +268,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
expectedKMSCloseGracePeriod = 42 * time.Second expectedKMSCloseGracePeriod = 42 * time.Second
correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml" correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml"
kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false) kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false, "")
if err != nil { if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First) t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First)
} }
@ -343,7 +343,7 @@ func TestKMSv1Deprecation(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, testCase.kmsv1Enabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, testCase.kmsv1Enabled)()
kmsv1Config := "testdata/valid-configs/kms/multiple-providers.yaml" kmsv1Config := "testdata/valid-configs/kms/multiple-providers.yaml"
_, err := LoadEncryptionConfig(testContext(t), kmsv1Config, false) _, err := LoadEncryptionConfig(testContext(t), kmsv1Config, false, "")
if !strings.Contains(errString(err), testCase.expectedErr) { if !strings.Contains(errString(err), testCase.expectedErr) {
t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err)) t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err))
} }
@ -384,7 +384,7 @@ func TestKMSvsEnablement(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, testCase.kmsv2Enabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, testCase.kmsv2Enabled)()
_, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false) _, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false, "")
if !strings.Contains(errString(err), testCase.expectedErr) { if !strings.Contains(errString(err), testCase.expectedErr) {
t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err)) t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err))
@ -486,7 +486,7 @@ func TestKMSvsEnablement(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel this upfront so the kms v2 checks do not block cancel() // cancel this upfront so the kms v2 checks do not block
_, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &tt.config) _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &tt.config, "")
if err == nil { if err == nil {
if kmsUsed == nil || kmsUsed.v2Used != tt.wantV2Used { if kmsUsed == nil || kmsUsed.v2Used != tt.wantV2Used {
t.Fatalf("unexpected kmsUsed value, expected: %v, got: %v", tt.wantV2Used, kmsUsed) t.Fatalf("unexpected kmsUsed value, expected: %v, got: %v", tt.wantV2Used, kmsUsed)
@ -729,7 +729,7 @@ func TestKMSMaxTimeout(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel this upfront so the kms v2 checks do not block cancel() // cancel this upfront so the kms v2 checks do not block
_, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &testCase.config) _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &testCase.config, "")
if !strings.Contains(errString(err), testCase.expectedErr) { if !strings.Contains(errString(err), testCase.expectedErr) {
t.Fatalf("expecting error calling prefixTransformersAndProbes, expected: %s, got: %s", testCase.expectedErr, errString(err)) t.Fatalf("expecting error calling prefixTransformersAndProbes, expected: %s, got: %s", testCase.expectedErr, errString(err))
@ -754,6 +754,7 @@ func TestKMSPluginHealthz(t *testing.T) {
kmsv2Probe := &kmsv2PluginProbe{ kmsv2Probe := &kmsv2PluginProbe{
name: "foo", name: "foo",
ttl: 3 * time.Second, ttl: 3 * time.Second,
apiServerID: "",
} }
keyID := "1" keyID := "1"
kmsv2Probe.state.Store(&envelopekmsv2.State{EncryptedObject: kmstypes.EncryptedObject{KeyID: keyID}}) kmsv2Probe.state.Store(&envelopekmsv2.State{EncryptedObject: kmstypes.EncryptedObject{KeyID: keyID}})
@ -853,7 +854,7 @@ func TestKMSPluginHealthz(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel this upfront so the kms v2 healthz check poll does not run cancel() // cancel this upfront so the kms v2 healthz check poll does not run
_, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config) _, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, "")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1298,7 +1299,7 @@ func TestWildcardMasking(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel) t.Cleanup(cancel)
_, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config) _, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "")
if errString(err) != tc.expectedError { if errString(err) != tc.expectedError {
t.Errorf("expected error %s but got %s", tc.expectedError, errString(err)) t.Errorf("expected error %s but got %s", tc.expectedError, errString(err))
} }
@ -1453,7 +1454,7 @@ func TestWildcardStructure(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel) t.Cleanup(cancel)
transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config) transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "")
if errString(err) != tc.errorValue { if errString(err) != tc.errorValue {
t.Errorf("expected error %s but got %s", tc.errorValue, errString(err)) t.Errorf("expected error %s but got %s", tc.errorValue, errString(err))
} }
@ -1724,7 +1725,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin
ctx := testContext(t) ctx := testContext(t)
t.Helper() t.Helper()
encryptionConfiguration, err := LoadEncryptionConfig(ctx, encryptionConfigPath, false) encryptionConfiguration, err := LoadEncryptionConfig(ctx, encryptionConfigPath, false, "")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -2123,6 +2124,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
&testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called
"panda", "panda",
h.getCurrentState, h.getCurrentState,
"",
) )
dataCtx := value.DefaultContext(sampleContextText) dataCtx := value.DefaultContext(sampleContextText)

15
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go
View File

@ -50,6 +50,13 @@ type DynamicKMSEncryptionConfigContent struct {
// dynamicTransformers updates the transformers when encryption config file changes. // dynamicTransformers updates the transformers when encryption config file changes.
dynamicTransformers *encryptionconfig.DynamicTransformers dynamicTransformers *encryptionconfig.DynamicTransformers
// identity of the api server
apiServerID string
}
func init() {
metrics.RegisterMetrics()
} }
// NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file. // NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
@ -57,6 +64,7 @@ func NewDynamicEncryptionConfiguration(
name, filePath string, name, filePath string,
dynamicTransformers *encryptionconfig.DynamicTransformers, dynamicTransformers *encryptionconfig.DynamicTransformers,
configContentHash string, configContentHash string,
apiServerID string,
) *DynamicKMSEncryptionConfigContent { ) *DynamicKMSEncryptionConfigContent {
encryptionConfig := &DynamicKMSEncryptionConfigContent{ encryptionConfig := &DynamicKMSEncryptionConfigContent{
name: name, name: name,
@ -64,6 +72,7 @@ func NewDynamicEncryptionConfiguration(
lastLoadedEncryptionConfigHash: configContentHash, lastLoadedEncryptionConfigHash: configContentHash,
dynamicTransformers: dynamicTransformers, dynamicTransformers: dynamicTransformers,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name), queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
apiServerID: apiServerID,
} }
encryptionConfig.queue.Add(workqueueKey) // to avoid missing any file changes that occur in between the initial load and Run encryptionConfig.queue.Add(workqueueKey) // to avoid missing any file changes that occur in between the initial load and Run
@ -172,11 +181,11 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
} }
if updatedEffectiveConfig && err == nil { if updatedEffectiveConfig && err == nil {
metrics.RecordEncryptionConfigAutomaticReloadSuccess() metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
} }
if err != nil { if err != nil {
metrics.RecordEncryptionConfigAutomaticReloadFailure() metrics.RecordEncryptionConfigAutomaticReloadFailure(d.apiServerID)
utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err)) utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
// add dummy item back to the queue to trigger file content processing. // add dummy item back to the queue to trigger file content processing.
d.queue.AddRateLimited(key) d.queue.AddRateLimited(key)
@ -224,7 +233,7 @@ func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.
err error, err error,
) { ) {
// this code path will only execute if reload=true. So passing true explicitly. // this code path will only execute if reload=true. So passing true explicitly.
encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true) encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true, d.apiServerID)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }

2
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go
View File

@ -46,6 +46,7 @@ func TestProcessEncryptionConfig(t *testing.T) {
testCase.filePath, testCase.filePath,
nil, nil,
"", "",
"",
) )
_, _, err := d.processEncryptionConfig(ctx) _, _, err := d.processEncryptionConfig(ctx)
@ -110,6 +111,7 @@ func TestWatchEncryptionConfigFile(t *testing.T) {
testFilePath, testFilePath,
nil, nil,
"", "",
"",
) )
errs := make(chan error, 1) errs := make(chan error, 1)

53
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go
View File

@ -17,6 +17,9 @@ limitations under the License.
package metrics package metrics
import ( import (
"crypto/sha256"
"fmt"
"hash"
"sync" "sync"
"k8s.io/component-base/metrics" "k8s.io/component-base/metrics"
@ -29,24 +32,26 @@ const (
) )
var ( var (
encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter( encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
&metrics.CounterOpts{ &metrics.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "automatic_reload_failures_total", Name: "automatic_reload_failures_total",
Help: "Total number of failed automatic reloads of encryption configuration.", Help: "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
[]string{"apiserver_id_hash"},
) )
encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter( encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
&metrics.CounterOpts{ &metrics.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "automatic_reload_success_total", Name: "automatic_reload_success_total",
Help: "Total number of successful automatic reloads of encryption configuration.", Help: "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
[]string{"apiserver_id_hash"},
) )
encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec( encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
@ -54,33 +59,53 @@ var (
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "automatic_reload_last_timestamp_seconds", Name: "automatic_reload_last_timestamp_seconds",
Help: "Timestamp of the last successful or failed automatic reload of encryption configuration.", Help: "Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.",
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
[]string{"status"}, []string{"status", "apiserver_id_hash"},
) )
) )
var registerMetrics sync.Once var registerMetrics sync.Once
var hashPool *sync.Pool
func RegisterMetrics() { func RegisterMetrics() {
registerMetrics.Do(func() { registerMetrics.Do(func() {
hashPool = &sync.Pool{
New: func() interface{} {
return sha256.New()
},
}
legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal) legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal) legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds) legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
}) })
} }
func RecordEncryptionConfigAutomaticReloadFailure() { func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
encryptionConfigAutomaticReloadFailureTotal.Inc() apiServerIDHash := getHash(apiServerID)
recordEncryptionConfigAutomaticReloadTimestamp("failure") encryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
} }
func RecordEncryptionConfigAutomaticReloadSuccess() { func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
encryptionConfigAutomaticReloadSuccessTotal.Inc() apiServerIDHash := getHash(apiServerID)
recordEncryptionConfigAutomaticReloadTimestamp("success") encryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
} }
func recordEncryptionConfigAutomaticReloadTimestamp(result string) { func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime() encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
}
func getHash(data string) string {
if len(data) == 0 {
return ""
}
h := hashPool.Get().(hash.Hash)
h.Reset()
h.Write([]byte(data))
dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
hashPool.Put(h)
return dataHash
} }

27
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go
View File

@ -24,11 +24,16 @@ import (
"k8s.io/component-base/metrics/testutil" "k8s.io/component-base/metrics/testutil"
) )
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
)
func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) { func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
expectedValue := ` expectedValue := `
# HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration. # HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter # TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter
apiserver_encryption_config_controller_automatic_reload_failures_total 1 apiserver_encryption_config_controller_automatic_reload_failures_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"} 1
` `
metrics := []string{ metrics := []string{
namespace + "_" + subsystem + "_automatic_reload_failures_total", namespace + "_" + subsystem + "_automatic_reload_failures_total",
@ -37,7 +42,7 @@ func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
encryptionConfigAutomaticReloadFailureTotal.Reset() encryptionConfigAutomaticReloadFailureTotal.Reset()
RegisterMetrics() RegisterMetrics()
RecordEncryptionConfigAutomaticReloadFailure() RecordEncryptionConfigAutomaticReloadFailure(testAPIServerID)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -45,9 +50,9 @@ func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) { func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
expectedValue := ` expectedValue := `
# HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration. # HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter # TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter
apiserver_encryption_config_controller_automatic_reload_success_total 1 apiserver_encryption_config_controller_automatic_reload_success_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"} 1
` `
metrics := []string{ metrics := []string{
namespace + "_" + subsystem + "_automatic_reload_success_total", namespace + "_" + subsystem + "_automatic_reload_success_total",
@ -56,7 +61,7 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
encryptionConfigAutomaticReloadSuccessTotal.Reset() encryptionConfigAutomaticReloadSuccessTotal.Reset()
RegisterMetrics() RegisterMetrics()
RecordEncryptionConfigAutomaticReloadSuccess() RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -70,18 +75,18 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}{ }{
{ {
expectedValue: ` expectedValue: `
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration. # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="failure"} 1.689101941e+09 apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="failure"} 1.689101941e+09
`, `,
resultLabel: "failure", resultLabel: "failure",
timestamp: 1689101941, timestamp: 1689101941,
}, },
{ {
expectedValue: ` expectedValue: `
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration. # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="success"} 1.689101941e+09 apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1.689101941e+09
`, `,
resultLabel: "success", resultLabel: "success",
timestamp: 1689101941, timestamp: 1689101941,
@ -95,7 +100,7 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
for _, tc := range testCases { for _, tc := range testCases {
encryptionConfigAutomaticReloadLastTimestampSeconds.Reset() encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel).Set(float64(tc.timestamp)) encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel, testAPIServerIDHash).Set(float64(tc.timestamp))
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil {
t.Fatal(err) t.Fatal(err)

3
staging/src/k8s.io/apiserver/pkg/server/options/etcd.go
View File

@ -294,7 +294,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
} }
}() }()
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload) encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID)
if err != nil { if err != nil {
return err return err
} }
@ -318,6 +318,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigFilepath,
dynamicTransformers, dynamicTransformers,
encryptionConfiguration.EncryptionFileContentHash, encryptionConfiguration.EncryptionFileContentHash,
c.APIServerID,
) )
go dynamicEncryptionConfigController.Run(ctxServer) go dynamicEncryptionConfigController.Run(ctxServer)

12
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go
View File

@ -111,21 +111,23 @@ type envelopeTransformer struct {
// cache is a thread-safe expiring lru cache which caches decrypted DEKs indexed by their encrypted form. // cache is a thread-safe expiring lru cache which caches decrypted DEKs indexed by their encrypted form.
cache *simpleCache cache *simpleCache
apiServerID string
} }
// NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme. // NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme.
// It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
// the data items they encrypt. // the data items they encrypt.
func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc) value.Transformer { func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string) value.Transformer {
return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, cacheTTL, clock.RealClock{}) return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, apiServerID, cacheTTL, clock.RealClock{})
} }
func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, cacheTTL time.Duration, clock clock.Clock) value.Transformer { func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string, cacheTTL time.Duration, clock clock.Clock) value.Transformer {
return &envelopeTransformer{ return &envelopeTransformer{
envelopeService: envelopeService, envelopeService: envelopeService,
providerName: providerName, providerName: providerName,
stateFunc: stateFunc, stateFunc: stateFunc,
cache: newSimpleCache(clock, cacheTTL, providerName), cache: newSimpleCache(clock, cacheTTL, providerName),
apiServerID: apiServerID,
} }
} }
@ -178,7 +180,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
return nil, false, err return nil, false, err
} }
} }
metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID) metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID, t.apiServerID)
out, stale, err := transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx) out, stale, err := transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx)
if err != nil { if err != nil {
@ -218,7 +220,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
return nil, err return nil, err
} }
metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID) metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID, t.apiServerID)
encObjectCopy := state.EncryptedObject encObjectCopy := state.EncryptedObject
encObjectCopy.EncryptedData = result encObjectCopy.EncryptedData = result

31
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go
View File

@ -53,6 +53,8 @@ const (
testContextText = "0123456789" testContextText = "0123456789"
testKeyHash = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" testKeyHash = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
testKeyVersion = "1" testKeyVersion = "1"
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
) )
// testEnvelopeService is a mock Envelope service which can be used to simulate remote Envelope services // testEnvelopeService is a mock Envelope service which can be used to simulate remote Envelope services
@ -178,7 +180,7 @@ func TestEnvelopeCaching(t *testing.T) {
} }
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
func() (State, error) { return state, nil }, func() (State, error) { return state, nil }, testAPIServerID,
tt.cacheTTL, fakeClock) tt.cacheTTL, fakeClock)
dataCtx := value.DefaultContext(testContextText) dataCtx := value.DefaultContext(testContextText)
@ -319,7 +321,7 @@ func TestEnvelopeTransformerStaleness(t *testing.T) {
var stateErr error var stateErr error
transformer := NewEnvelopeTransformer(envelopeService, testProviderName, transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
func() (State, error) { return state, stateErr }, func() (State, error) { return state, stateErr }, testAPIServerID,
) )
dataCtx := value.DefaultContext(testContextText) dataCtx := value.DefaultContext(testContextText)
@ -376,7 +378,7 @@ func TestEnvelopeTransformerStateFunc(t *testing.T) {
stateErr := fmt.Errorf("some state error") stateErr := fmt.Errorf("some state error")
transformer := NewEnvelopeTransformer(envelopeService, testProviderName, transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
func() (State, error) { return state, stateErr }, func() (State, error) { return state, stateErr }, testAPIServerID,
) )
dataCtx := value.DefaultContext(testContextText) dataCtx := value.DefaultContext(testContextText)
@ -513,6 +515,7 @@ func TestTransformToStorageError(t *testing.T) {
envelopeService.SetAnnotations(tt.annotations) envelopeService.SetAnnotations(tt.annotations)
transformer := NewEnvelopeTransformer(envelopeService, testProviderName, transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool()), testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool()),
testAPIServerID,
) )
dataCtx := value.DefaultContext(testContextText) dataCtx := value.DefaultContext(testContextText)
@ -838,6 +841,7 @@ func TestEnvelopeMetrics(t *testing.T) {
envelopeService := newTestEnvelopeService() envelopeService := newTestEnvelopeService()
transformer := NewEnvelopeTransformer(envelopeService, testProviderName, transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
testStateFunc(testContext(t), envelopeService, clock.RealClock{}, randomBool()), testStateFunc(testContext(t), envelopeService, clock.RealClock{}, randomBool()),
testAPIServerID,
) )
dataCtx := value.DefaultContext(testContextText) dataCtx := value.DefaultContext(testContextText)
@ -859,11 +863,11 @@ func TestEnvelopeMetrics(t *testing.T) {
"apiserver_envelope_encryption_key_id_hash_total", "apiserver_envelope_encryption_key_id_hash_total",
}, },
want: fmt.Sprintf(` want: fmt.Sprintf(`
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
# TYPE apiserver_envelope_encryption_key_id_hash_total counter # TYPE apiserver_envelope_encryption_key_id_hash_total counter
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
`, testKeyHash, testProviderName, metrics.FromStorageLabel, testKeyHash, testProviderName, metrics.ToStorageLabel), `, testAPIServerIDHash, testKeyHash, testProviderName, metrics.FromStorageLabel, testAPIServerIDHash, testKeyHash, testProviderName, metrics.ToStorageLabel),
}, },
} }
@ -931,10 +935,10 @@ func TestEnvelopeMetricsCache(t *testing.T) {
transformer1 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { transformer1 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) {
// return different states to ensure we get expected number of cache keys after restart on decryption // return different states to ensure we get expected number of cache keys after restart on decryption
return testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool())() return testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool())()
}) }, testAPIServerID)
transformer2 := NewEnvelopeTransformer(envelopeService, provider2, func() (State, error) { return state, nil }) transformer2 := NewEnvelopeTransformer(envelopeService, provider2, func() (State, error) { return state, nil }, testAPIServerID)
// used for restart // used for restart
transformer3 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { return state, nil }) transformer3 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { return state, nil }, testAPIServerID)
var transformedDatas [][]byte var transformedDatas [][]byte
for j := 0; j < numOfStates; j++ { for j := 0; j < numOfStates; j++ {
transformedData, err := transformer1.TransformToStorage(ctx, []byte(testText), dataCtx) transformedData, err := transformer1.TransformToStorage(ctx, []byte(testText), dataCtx)
@ -1029,8 +1033,7 @@ func TestEnvelopeLogging(t *testing.T) {
envelopeService := newTestEnvelopeService() envelopeService := newTestEnvelopeService()
fakeClock := testingclock.NewFakeClock(time.Now()) fakeClock := testingclock.NewFakeClock(time.Now())
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
testStateFunc(tc.ctx, envelopeService, clock.RealClock{}, randomBool()), testStateFunc(tc.ctx, envelopeService, clock.RealClock{}, randomBool()), testAPIServerID, 1*time.Second, fakeClock)
1*time.Second, fakeClock)
dataCtx := value.DefaultContext([]byte(testContextText)) dataCtx := value.DefaultContext([]byte(testContextText))
originalText := []byte(testText) originalText := []byte(testText)
@ -1082,7 +1085,7 @@ func TestCacheNotCorrupted(t *testing.T) {
} }
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
func() (State, error) { return state, nil }, func() (State, error) { return state, nil }, testAPIServerID,
1*time.Second, fakeClock) 1*time.Second, fakeClock)
dataCtx := value.DefaultContext(testContextText) dataCtx := value.DefaultContext(testContextText)
@ -1108,7 +1111,7 @@ func TestCacheNotCorrupted(t *testing.T) {
} }
transformer = newEnvelopeTransformerWithClock(envelopeService, testProviderName, transformer = newEnvelopeTransformerWithClock(envelopeService, testProviderName,
func() (State, error) { return state, nil }, func() (State, error) { return state, nil }, testAPIServerID,
1*time.Second, fakeClock) 1*time.Second, fakeClock)
transformedData2, err := transformer.TransformToStorage(ctx, originalText, dataCtx) transformedData2, err := transformer.TransformToStorage(ctx, originalText, dataCtx)

60
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go
View File

@ -44,6 +44,7 @@ type metricLabels struct {
transformationType string transformationType string
providerName string providerName string
keyIDHash string keyIDHash string
apiServerIDHash string
} }
/* /*
@ -107,21 +108,21 @@ var (
// keyIDHashTotal is the number of times a keyID is used // keyIDHashTotal is the number of times a keyID is used
// e.g. apiserver_envelope_encryption_key_id_hash_total counter // e.g. apiserver_envelope_encryption_key_id_hash_total counter
// apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256", // apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256",key_id_hash="sha256",
// provider_name="providerName",transformation_type="from_storage"} 1 // provider_name="providerName",transformation_type="from_storage"} 1
KeyIDHashTotal = metrics.NewCounterVec( KeyIDHashTotal = metrics.NewCounterVec(
&metrics.CounterOpts{ &metrics.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "key_id_hash_total", Name: "key_id_hash_total",
Help: "Number of times a keyID is used split by transformation type and provider.", Help: "Number of times a keyID is used split by transformation type, provider, and apiserver identity.",
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
[]string{"transformation_type", "provider_name", "key_id_hash"}, []string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"},
) )
// keyIDHashLastTimestampSeconds is the last time in seconds when a keyID was used // keyIDHashLastTimestampSeconds is the last time in seconds when a keyID was used
// e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09 // e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09
KeyIDHashLastTimestampSeconds = metrics.NewGaugeVec( KeyIDHashLastTimestampSeconds = metrics.NewGaugeVec(
&metrics.GaugeOpts{ &metrics.GaugeOpts{
Namespace: namespace, Namespace: namespace,
@ -130,11 +131,11 @@ var (
Help: "The last time in seconds when a keyID was used.", Help: "The last time in seconds when a keyID was used.",
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
[]string{"transformation_type", "provider_name", "key_id_hash"}, []string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"},
) )
// keyIDHashStatusLastTimestampSeconds is the last time in seconds when a keyID was returned by the Status RPC call. // keyIDHashStatusLastTimestampSeconds is the last time in seconds when a keyID was returned by the Status RPC call.
// e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09 // e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09
KeyIDHashStatusLastTimestampSeconds = metrics.NewGaugeVec( KeyIDHashStatusLastTimestampSeconds = metrics.NewGaugeVec(
&metrics.GaugeOpts{ &metrics.GaugeOpts{
Namespace: namespace, Namespace: namespace,
@ -143,7 +144,7 @@ var (
Help: "The last time in seconds when a keyID was returned by the Status RPC call.", Help: "The last time in seconds when a keyID was returned by the Status RPC call.",
StabilityLevel: metrics.ALPHA, StabilityLevel: metrics.ALPHA,
}, },
[]string{"provider_name", "key_id_hash"}, []string{"provider_name", "key_id_hash", "apiserver_id_hash"},
) )
InvalidKeyIDFromStatusTotal = metrics.NewCounterVec( InvalidKeyIDFromStatusTotal = metrics.NewCounterVec(
@ -182,19 +183,19 @@ func registerLRUMetrics() {
keyIDHashTotalMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) { keyIDHashTotalMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
item := key.(metricLabels) item := key.(metricLabels)
if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted { if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
klog.InfoS("Deleted keyIDHashTotalMetricLabels", "transformationType", item.transformationType, klog.InfoS("Deleted keyIDHashTotalMetricLabels", "transformationType", item.transformationType,
"providerName", item.providerName, "keyIDHash", item.keyIDHash) "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
} }
if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted { if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
klog.InfoS("Deleted keyIDHashLastTimestampSecondsMetricLabels", "transformationType", item.transformationType, klog.InfoS("Deleted keyIDHashLastTimestampSecondsMetricLabels", "transformationType", item.transformationType,
"providerName", item.providerName, "keyIDHash", item.keyIDHash) "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
} }
}) })
keyIDHashStatusLastTimestampSecondsMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) { keyIDHashStatusLastTimestampSecondsMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
item := key.(metricLabels) item := key.(metricLabels)
if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash); deleted { if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash) klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
} }
}) })
} }
@ -218,22 +219,22 @@ func RegisterMetrics() {
} }
// RecordKeyID records total count and last time in seconds when a KeyID was used for TransformFromStorage and TransformToStorage operations // RecordKeyID records total count and last time in seconds when a KeyID was used for TransformFromStorage and TransformToStorage operations
func RecordKeyID(transformationType, providerName, keyID string) { func RecordKeyID(transformationType, providerName, keyID, apiServerID string) {
lockRecordKeyID.Lock() lockRecordKeyID.Lock()
defer lockRecordKeyID.Unlock() defer lockRecordKeyID.Unlock()
keyIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID) keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID, apiServerID)
KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash).Inc() KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).Inc()
KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash).SetToCurrentTime() KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).SetToCurrentTime()
} }
// RecordKeyIDFromStatus records last time in seconds when a KeyID was returned by the Status RPC call. // RecordKeyIDFromStatus records last time in seconds when a KeyID was returned by the Status RPC call.
func RecordKeyIDFromStatus(providerName, keyID string) { func RecordKeyIDFromStatus(providerName, keyID, apiServerID string) {
lockRecordKeyIDStatus.Lock() lockRecordKeyIDStatus.Lock()
defer lockRecordKeyIDStatus.Unlock() defer lockRecordKeyIDStatus.Unlock()
keyIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID) keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID, apiServerID)
KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash).SetToCurrentTime() KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash, apiServerIDHash).SetToCurrentTime()
} }
func RecordInvalidKeyIDFromStatus(providerName, errCode string) { func RecordInvalidKeyIDFromStatus(providerName, errCode string) {
@ -297,24 +298,25 @@ func getErrorCode(err error) string {
} }
func getHash(data string) string { func getHash(data string) string {
if len(data) == 0 {
return ""
}
h := hashPool.Get().(hash.Hash) h := hashPool.Get().(hash.Hash)
h.Reset() h.Reset()
h.Write([]byte(data)) h.Write([]byte(data))
result := fmt.Sprintf("sha256:%x", h.Sum(nil)) dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
hashPool.Put(h) hashPool.Put(h)
return result return dataHash
} }
func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID string) string { func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID, apiServerID string) (string, string) {
keyIDHash := "" keyIDHash := getHash(keyID)
// only get hash if the keyID is not empty apiServerIDHash := getHash(apiServerID)
if len(keyID) > 0 {
keyIDHash = getHash(keyID)
}
c.Add(metricLabels{ c.Add(metricLabels{
transformationType: transformationType, transformationType: transformationType,
providerName: providerName, providerName: providerName,
keyIDHash: keyIDHash, keyIDHash: keyIDHash,
apiServerIDHash: apiServerIDHash,
}, nil) // value is irrelevant, this is a set and not a map }, nil) // value is irrelevant, this is a set and not a map
return keyIDHash return keyIDHash, apiServerIDHash
} }

72
staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics_test.go
View File

@ -36,6 +36,8 @@ const (
testKeyHash2 = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" testKeyHash2 = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35"
testKeyHash3 = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" testKeyHash3 = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce"
testProviderNameForMetric = "providerName" testProviderNameForMetric = "providerName"
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
) )
var ( var (
@ -204,6 +206,7 @@ func TestRecordKeyID_Serial(t *testing.T) {
metrics []string metrics []string
providerName string providerName string
transformationType string transformationType string
apiServerID string
want string want string
}{ }{
{ {
@ -214,11 +217,12 @@ func TestRecordKeyID_Serial(t *testing.T) {
}, },
providerName: testProviderNameForMetric, providerName: testProviderNameForMetric,
transformationType: FromStorageLabel, transformationType: FromStorageLabel,
apiServerID: testAPIServerID,
want: fmt.Sprintf(` want: fmt.Sprintf(`
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
# TYPE apiserver_envelope_encryption_key_id_hash_total counter # TYPE apiserver_envelope_encryption_key_id_hash_total counter
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
`, testKeyHash1, testProviderNameForMetric, FromStorageLabel), `, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel),
}, },
{ {
desc: "keyIDHash total more labels", desc: "keyIDHash total more labels",
@ -228,12 +232,13 @@ func TestRecordKeyID_Serial(t *testing.T) {
}, },
providerName: testProviderNameForMetric, providerName: testProviderNameForMetric,
transformationType: FromStorageLabel, transformationType: FromStorageLabel,
apiServerID: testAPIServerID,
want: fmt.Sprintf(` want: fmt.Sprintf(`
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
# TYPE apiserver_envelope_encryption_key_id_hash_total counter # TYPE apiserver_envelope_encryption_key_id_hash_total counter
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
`, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testKeyHash2, testProviderNameForMetric, FromStorageLabel), `, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel),
}, },
{ {
desc: "keyIDHash total same labels", desc: "keyIDHash total same labels",
@ -243,12 +248,13 @@ func TestRecordKeyID_Serial(t *testing.T) {
}, },
providerName: testProviderNameForMetric, providerName: testProviderNameForMetric,
transformationType: FromStorageLabel, transformationType: FromStorageLabel,
apiServerID: testAPIServerID,
want: fmt.Sprintf(` want: fmt.Sprintf(`
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
# TYPE apiserver_envelope_encryption_key_id_hash_total counter # TYPE apiserver_envelope_encryption_key_id_hash_total counter
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2
`, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testKeyHash2, testProviderNameForMetric, FromStorageLabel), `, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel),
}, },
{ {
desc: "keyIDHash total exceeds limit, remove first label, and empty keyID", desc: "keyIDHash total exceeds limit, remove first label, and empty keyID",
@ -258,12 +264,29 @@ func TestRecordKeyID_Serial(t *testing.T) {
}, },
providerName: testProviderNameForMetric, providerName: testProviderNameForMetric,
transformationType: FromStorageLabel, transformationType: FromStorageLabel,
apiServerID: testAPIServerID,
want: fmt.Sprintf(` want: fmt.Sprintf(`
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
# TYPE apiserver_envelope_encryption_key_id_hash_total counter # TYPE apiserver_envelope_encryption_key_id_hash_total counter
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
`, testKeyHash2, testProviderNameForMetric, FromStorageLabel, "", testProviderNameForMetric, FromStorageLabel), `, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, "", testProviderNameForMetric, FromStorageLabel),
},
{
desc: "keyIDHash total exceeds limit, remove first label, empty keyID, and empty testAPIServerID",
keyID: "",
metrics: []string{
"apiserver_envelope_encryption_key_id_hash_total",
},
providerName: testProviderNameForMetric,
transformationType: FromStorageLabel,
apiServerID: "",
want: fmt.Sprintf(`
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
`, testAPIServerIDHash, "", testProviderNameForMetric, FromStorageLabel, "", "", testProviderNameForMetric, FromStorageLabel),
}, },
{ {
desc: "keyIDHash total exceeds limit 2, remove first label", desc: "keyIDHash total exceeds limit 2, remove first label",
@ -273,12 +296,13 @@ func TestRecordKeyID_Serial(t *testing.T) {
}, },
providerName: testProviderNameForMetric, providerName: testProviderNameForMetric,
transformationType: FromStorageLabel, transformationType: FromStorageLabel,
apiServerID: "",
want: fmt.Sprintf(` want: fmt.Sprintf(`
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider. # HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
# TYPE apiserver_envelope_encryption_key_id_hash_total counter # TYPE apiserver_envelope_encryption_key_id_hash_total counter
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1 apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
`, "", testProviderNameForMetric, FromStorageLabel, testKeyHash1, testProviderNameForMetric, FromStorageLabel), `, "", "", testProviderNameForMetric, FromStorageLabel, "", testKeyHash1, testProviderNameForMetric, FromStorageLabel),
}, },
} }
@ -289,7 +313,7 @@ func TestRecordKeyID_Serial(t *testing.T) {
for _, tt := range testCases { for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) { t.Run(tt.desc, func(t *testing.T) {
RecordKeyID(tt.transformationType, tt.providerName, tt.keyID) RecordKeyID(tt.transformationType, tt.providerName, tt.keyID, tt.apiServerID)
// We are not resetting the metric here as each test is not independent in order to validate the behavior // We are not resetting the metric here as each test is not independent in order to validate the behavior
// when the metric labels exceed the limit to ensure the labels are not unbounded. // when the metric labels exceed the limit to ensure the labels are not unbounded.
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
@ -313,12 +337,14 @@ func TestRecordKeyIDLRUKey(t *testing.T) {
go func() { go func() {
defer wg.Done() defer wg.Done()
keyID := rand.String(32) keyID := rand.String(32)
apiServerID := rand.String(32)
key := metricLabels{ key := metricLabels{
transformationType: rand.String(32), transformationType: rand.String(32),
providerName: rand.String(32), providerName: rand.String(32),
keyIDHash: getHash(keyID), keyIDHash: getHash(keyID),
apiServerIDHash: getHash(apiServerID),
} }
RecordKeyID(key.transformationType, key.providerName, keyID) RecordKeyID(key.transformationType, key.providerName, keyID, apiServerID)
}() }()
} }
wg.Wait() wg.Wait()
@ -359,11 +385,13 @@ func TestRecordKeyIDFromStatus(t *testing.T) {
go func() { go func() {
defer wg.Done() defer wg.Done()
keyID := rand.String(32) keyID := rand.String(32)
apiServerID := rand.String(32)
key := metricLabels{ key := metricLabels{
providerName: rand.String(32), providerName: rand.String(32),
keyIDHash: getHash(keyID), keyIDHash: getHash(keyID),
apiServerIDHash: getHash(apiServerID),
} }
RecordKeyIDFromStatus(key.providerName, keyID) RecordKeyIDFromStatus(key.providerName, keyID, apiServerID)
}() }()
} }
wg.Wait() wg.Wait()

58
test/integration/controlplane/transformation/kms_transformation_test.go
View File

@ -29,21 +29,27 @@ import (
"math/rand" "math/rand"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/google/go-cmp/cmp"
clientv3 "go.etcd.io/etcd/client/v3" clientv3 "go.etcd.io/etcd/client/v3"
"golang.org/x/crypto/cryptobyte" "golang.org/x/crypto/cryptobyte"
"k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/storage/value" "k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1" mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1"
"k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
featuregatetesting "k8s.io/component-base/featuregate/testing" featuregatetesting "k8s.io/component-base/featuregate/testing"
kmsapi "k8s.io/kms/apis/v1beta1" kmsapi "k8s.io/kms/apis/v1beta1"
"k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration"
@ -308,6 +314,8 @@ resources:
cachesize: 1000 cachesize: 1000
endpoint: unix:///@kms-provider.sock endpoint: unix:///@kms-provider.sock
` `
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
_ = mock.NewBase64Plugin(t, "@kms-provider.sock") _ = mock.NewBase64Plugin(t, "@kms-provider.sock")
var restarted bool var restarted bool
test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig) test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig)
@ -319,6 +327,26 @@ resources:
test.cleanUp() test.cleanUp()
} }
}() }()
ctx := testContext(t)
// the global metrics registry persists across test runs - reset it here so we can make assertions
copyConfig := rest.CopyConfig(test.kubeAPIServer.ClientConfig)
copyConfig.GroupVersion = &schema.GroupVersion{}
copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
rc, err := rest.RESTClientFor(copyConfig)
if err != nil {
t.Fatal(err)
}
if err := rc.Delete().AbsPath("/metrics").Do(ctx).Error(); err != nil {
t.Fatal(err)
}
// assert that the metrics we collect during the test run match expectations
// NOTE: 2 successful automatic reload resulted from 2 config file updates
wantMetricStrings := []string{
`apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_encryption_config_controller_automatic_reload_success_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795"} 2`,
}
test.secret, err = test.createSecret(testSecret, testNamespace) test.secret, err = test.createSecret(testSecret, testNamespace)
if err != nil { if err != nil {
@ -378,7 +406,7 @@ resources:
// run storage migration // run storage migration
// get secrets // get secrets
ctx := testContext(t)
secretsList, err := test.restClient.CoreV1().Secrets("").List( secretsList, err := test.restClient.CoreV1().Secrets("").List(
ctx, ctx,
metav1.ListOptions{}, metav1.ListOptions{},
@ -522,6 +550,34 @@ resources:
if _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}); err != nil { if _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}); err != nil {
t.Fatalf("failed to list configmaps, err: %v", err) t.Fatalf("failed to list configmaps, err: %v", err)
} }
// recreate rest client with the new transformTest
copyConfig = rest.CopyConfig(test.kubeAPIServer.ClientConfig)
copyConfig.GroupVersion = &schema.GroupVersion{}
copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
rc, err = rest.RESTClientFor(copyConfig)
if err != nil {
t.Fatal(err)
}
defer func() {
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
if err != nil {
t.Fatal(err)
}
var gotMetricStrings []string
trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`)
for _, line := range strings.Split(string(body), "\n") {
if strings.HasPrefix(line, "apiserver_encryption_config_controller_") {
if strings.Contains(line, "_seconds") {
line = trimFP.ReplaceAllString(line, `$1`) + "} FP" // ignore floating point metric values
}
gotMetricStrings = append(gotMetricStrings, line)
}
}
if diff := cmp.Diff(wantMetricStrings, gotMetricStrings); diff != "" {
t.Errorf("unexpected metrics diff (-want +got): %s", diff)
}
}()
} }
func TestEncryptAll(t *testing.T) { func TestEncryptAll(t *testing.T) {

10
test/integration/controlplane/transformation/kmsv2_transformation_test.go
View File

@ -199,7 +199,7 @@ resources:
name: kms-provider name: kms-provider
endpoint: unix:///@kms-provider.sock endpoint: unix:///@kms-provider.sock
` `
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
providerName := "kms-provider" providerName := "kms-provider"
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock") pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
@ -226,10 +226,10 @@ resources:
// assert that the metrics we collect during the test run match expectations // assert that the metrics we collect during the test run match expectations
wantMetricStrings := []string{ wantMetricStrings := []string{
`apiserver_envelope_encryption_dek_source_cache_size{provider_name="kms-provider"} 1`, `apiserver_envelope_encryption_dek_source_cache_size{provider_name="kms-provider"} 1`,
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`, `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`,
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`, `apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`,
`apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`, `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`,
`apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`, `apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`,
} }
defer func() { defer func() {
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx) body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)