mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-31 07:20:13 +00:00
kmsv2: add apiserver identity to metrics
Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>
This commit is contained in:
Rita Zhang
parent
37cf2638c9
commit
43ccf6c4e8
@ -43,12 +43,13 @@ import (
|
||||
"k8s.io/apiserver/pkg/apis/config/validation"
|
||||
"k8s.io/apiserver/pkg/features"
|
||||
"k8s.io/apiserver/pkg/server/healthz"
|
||||
"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
|
||||
storagevalue "k8s.io/apiserver/pkg/storage/value"
|
||||
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
|
||||
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
|
||||
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
|
||||
envelopemetrics "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
@ -111,6 +112,9 @@ func init() {
|
||||
utilruntime.Must(apiserverconfig.AddToScheme(configScheme))
|
||||
utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme))
|
||||
codecs = serializer.NewCodecFactory(configScheme)
|
||||
envelopemetrics.RegisterMetrics()
|
||||
storagevalue.RegisterMetrics()
|
||||
metrics.RegisterMetrics()
|
||||
}
|
||||
|
||||
type kmsPluginHealthzResponse struct {
|
||||
@ -133,6 +137,7 @@ type kmsv2PluginProbe struct {
|
||||
service kmsservice.Service
|
||||
lastResponse *kmsPluginHealthzResponse
|
||||
l *sync.Mutex
|
||||
apiServerID string
|
||||
}
|
||||
|
||||
type kmsHealthChecker []healthz.HealthChecker
|
||||
@ -186,13 +191,13 @@ type EncryptionConfiguration struct {
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
|
||||
func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*EncryptionConfiguration, error) {
|
||||
func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) {
|
||||
config, contentHash, err := loadConfig(filepath, reload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while parsing file: %w", err)
|
||||
}
|
||||
|
||||
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config)
|
||||
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while building transformers: %w", err)
|
||||
}
|
||||
@ -217,9 +222,9 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E
|
||||
// getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
|
||||
func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
|
||||
var kmsHealthChecks []healthz.HealthChecker
|
||||
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config)
|
||||
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@ -238,7 +243,7 @@ type healthChecker interface {
|
||||
// getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
|
||||
func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
|
||||
resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
|
||||
var probes []healthChecker
|
||||
var kmsUsed kmsState
|
||||
@ -247,7 +252,7 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
|
||||
for _, resourceConfig := range config.Resources {
|
||||
resourceConfig := resourceConfig
|
||||
|
||||
transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig)
|
||||
transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@ -454,10 +459,10 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
|
||||
}
|
||||
|
||||
if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
|
||||
metrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
|
||||
envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
|
||||
errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
|
||||
} else {
|
||||
metrics.RecordKeyIDFromStatus(h.name, response.KeyID)
|
||||
envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID)
|
||||
// unconditionally append as we filter out nil errors below
|
||||
errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID())))
|
||||
}
|
||||
@ -499,7 +504,7 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
|
||||
// prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
|
||||
func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
|
||||
var transformers []storagevalue.PrefixTransformer
|
||||
var probes []healthChecker
|
||||
var kmsUsed kmsState
|
||||
@ -527,7 +532,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
|
||||
transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
|
||||
|
||||
case provider.KMS != nil:
|
||||
transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS)
|
||||
transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID)
|
||||
if transformerErr == nil {
|
||||
probes = append(probes, probe)
|
||||
kmsUsed.accumulate(used)
|
||||
@ -686,7 +691,7 @@ func (s *kmsState) accumulate(other *kmsState) {
|
||||
// kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
|
||||
func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
|
||||
kmsName := config.Name
|
||||
switch config.APIVersion {
|
||||
case kmsAPIVersionV1:
|
||||
@ -732,14 +737,14 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
|
||||
service: envelopeService,
|
||||
l: &sync.Mutex{},
|
||||
lastResponse: &kmsPluginHealthzResponse{},
|
||||
apiServerID: apiServerID,
|
||||
}
|
||||
// initialize state so that Load always works
|
||||
probe.state.Store(&envelopekmsv2.State{})
|
||||
|
||||
primeAndProbeKMSv2(ctx, probe, kmsName)
|
||||
|
||||
transformer := storagevalue.PrefixTransformer{
|
||||
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState),
|
||||
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID),
|
||||
Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"),
|
||||
}
|
||||
|
||||
|
||||
@ -207,7 +207,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
|
||||
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
|
||||
expectedKMSCloseGracePeriod := 46 * time.Second
|
||||
correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml"
|
||||
identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false)
|
||||
identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst)
|
||||
}
|
||||
@ -218,7 +218,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
|
||||
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
|
||||
expectedKMSCloseGracePeriod = 32 * time.Second
|
||||
correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml"
|
||||
aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false)
|
||||
aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst)
|
||||
}
|
||||
@ -227,7 +227,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
|
||||
}
|
||||
|
||||
invalidConfigWithAesGcm := "testdata/invalid-configs/invalid-aes-gcm.yaml"
|
||||
_, err = LoadEncryptionConfig(ctx, invalidConfigWithAesGcm, false)
|
||||
_, err = LoadEncryptionConfig(ctx, invalidConfigWithAesGcm, false, "")
|
||||
if !strings.Contains(errString(err), "error while parsing file") {
|
||||
t.Fatalf("should result in error while parsing configuration file: %s.\nThe file was:\n%s", err, invalidConfigWithAesGcm)
|
||||
}
|
||||
@ -235,7 +235,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
|
||||
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
|
||||
expectedKMSCloseGracePeriod = 26 * time.Second
|
||||
correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml"
|
||||
aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false)
|
||||
aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst)
|
||||
}
|
||||
@ -246,7 +246,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
|
||||
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
|
||||
expectedKMSCloseGracePeriod = 14 * time.Second
|
||||
correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml"
|
||||
secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false)
|
||||
secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst)
|
||||
}
|
||||
@ -257,7 +257,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
|
||||
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
|
||||
expectedKMSCloseGracePeriod = 34 * time.Second
|
||||
correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml"
|
||||
kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false)
|
||||
kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst)
|
||||
}
|
||||
@ -268,7 +268,7 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
|
||||
// Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163
|
||||
expectedKMSCloseGracePeriod = 42 * time.Second
|
||||
correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml"
|
||||
kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false)
|
||||
kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First)
|
||||
}
|
||||
@ -343,7 +343,7 @@ func TestKMSv1Deprecation(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, testCase.kmsv1Enabled)()
|
||||
|
||||
kmsv1Config := "testdata/valid-configs/kms/multiple-providers.yaml"
|
||||
_, err := LoadEncryptionConfig(testContext(t), kmsv1Config, false)
|
||||
_, err := LoadEncryptionConfig(testContext(t), kmsv1Config, false, "")
|
||||
if !strings.Contains(errString(err), testCase.expectedErr) {
|
||||
t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err))
|
||||
}
|
||||
@ -384,7 +384,7 @@ func TestKMSvsEnablement(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)()
|
||||
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, testCase.kmsv2Enabled)()
|
||||
_, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false)
|
||||
_, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false, "")
|
||||
|
||||
if !strings.Contains(errString(err), testCase.expectedErr) {
|
||||
t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err))
|
||||
@ -486,7 +486,7 @@ func TestKMSvsEnablement(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel this upfront so the kms v2 checks do not block
|
||||
|
||||
_, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &tt.config)
|
||||
_, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &tt.config, "")
|
||||
if err == nil {
|
||||
if kmsUsed == nil || kmsUsed.v2Used != tt.wantV2Used {
|
||||
t.Fatalf("unexpected kmsUsed value, expected: %v, got: %v", tt.wantV2Used, kmsUsed)
|
||||
@ -729,7 +729,7 @@ func TestKMSMaxTimeout(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel this upfront so the kms v2 checks do not block
|
||||
|
||||
_, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &testCase.config)
|
||||
_, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &testCase.config, "")
|
||||
|
||||
if !strings.Contains(errString(err), testCase.expectedErr) {
|
||||
t.Fatalf("expecting error calling prefixTransformersAndProbes, expected: %s, got: %s", testCase.expectedErr, errString(err))
|
||||
@ -752,8 +752,9 @@ func TestKMSPluginHealthz(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
|
||||
|
||||
kmsv2Probe := &kmsv2PluginProbe{
|
||||
name: "foo",
|
||||
ttl: 3 * time.Second,
|
||||
name: "foo",
|
||||
ttl: 3 * time.Second,
|
||||
apiServerID: "",
|
||||
}
|
||||
keyID := "1"
|
||||
kmsv2Probe.state.Store(&envelopekmsv2.State{EncryptedObject: kmstypes.EncryptedObject{KeyID: keyID}})
|
||||
@ -853,7 +854,7 @@ func TestKMSPluginHealthz(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel this upfront so the kms v2 healthz check poll does not run
|
||||
_, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config)
|
||||
_, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -1298,7 +1299,7 @@ func TestWildcardMasking(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
_, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config)
|
||||
_, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "")
|
||||
if errString(err) != tc.expectedError {
|
||||
t.Errorf("expected error %s but got %s", tc.expectedError, errString(err))
|
||||
}
|
||||
@ -1453,7 +1454,7 @@ func TestWildcardStructure(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config)
|
||||
transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "")
|
||||
if errString(err) != tc.errorValue {
|
||||
t.Errorf("expected error %s but got %s", tc.errorValue, errString(err))
|
||||
}
|
||||
@ -1724,7 +1725,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin
|
||||
ctx := testContext(t)
|
||||
|
||||
t.Helper()
|
||||
encryptionConfiguration, err := LoadEncryptionConfig(ctx, encryptionConfigPath, false)
|
||||
encryptionConfiguration, err := LoadEncryptionConfig(ctx, encryptionConfigPath, false, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -2123,6 +2124,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
|
||||
&testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called
|
||||
"panda",
|
||||
h.getCurrentState,
|
||||
"",
|
||||
)
|
||||
|
||||
dataCtx := value.DefaultContext(sampleContextText)
|
||||
|
||||
@ -50,6 +50,13 @@ type DynamicKMSEncryptionConfigContent struct {
|
||||
|
||||
// dynamicTransformers updates the transformers when encryption config file changes.
|
||||
dynamicTransformers *encryptionconfig.DynamicTransformers
|
||||
|
||||
// identity of the api server
|
||||
apiServerID string
|
||||
}
|
||||
|
||||
func init() {
|
||||
metrics.RegisterMetrics()
|
||||
}
|
||||
|
||||
// NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
|
||||
@ -57,6 +64,7 @@ func NewDynamicEncryptionConfiguration(
|
||||
name, filePath string,
|
||||
dynamicTransformers *encryptionconfig.DynamicTransformers,
|
||||
configContentHash string,
|
||||
apiServerID string,
|
||||
) *DynamicKMSEncryptionConfigContent {
|
||||
encryptionConfig := &DynamicKMSEncryptionConfigContent{
|
||||
name: name,
|
||||
@ -64,6 +72,7 @@ func NewDynamicEncryptionConfiguration(
|
||||
lastLoadedEncryptionConfigHash: configContentHash,
|
||||
dynamicTransformers: dynamicTransformers,
|
||||
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
|
||||
apiServerID: apiServerID,
|
||||
}
|
||||
encryptionConfig.queue.Add(workqueueKey) // to avoid missing any file changes that occur in between the initial load and Run
|
||||
|
||||
@ -172,11 +181,11 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
|
||||
}
|
||||
|
||||
if updatedEffectiveConfig && err == nil {
|
||||
metrics.RecordEncryptionConfigAutomaticReloadSuccess()
|
||||
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
metrics.RecordEncryptionConfigAutomaticReloadFailure()
|
||||
metrics.RecordEncryptionConfigAutomaticReloadFailure(d.apiServerID)
|
||||
utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
|
||||
// add dummy item back to the queue to trigger file content processing.
|
||||
d.queue.AddRateLimited(key)
|
||||
@ -224,7 +233,7 @@ func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.
|
||||
err error,
|
||||
) {
|
||||
// this code path will only execute if reload=true. So passing true explicitly.
|
||||
encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true)
|
||||
encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true, d.apiServerID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
@ -46,6 +46,7 @@ func TestProcessEncryptionConfig(t *testing.T) {
|
||||
testCase.filePath,
|
||||
nil,
|
||||
"",
|
||||
"",
|
||||
)
|
||||
|
||||
_, _, err := d.processEncryptionConfig(ctx)
|
||||
@ -110,6 +111,7 @@ func TestWatchEncryptionConfigFile(t *testing.T) {
|
||||
testFilePath,
|
||||
nil,
|
||||
"",
|
||||
"",
|
||||
)
|
||||
|
||||
errs := make(chan error, 1)
|
||||
|
||||
@ -17,6 +17,9 @@ limitations under the License.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"hash"
|
||||
"sync"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
@ -29,24 +32,26 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter(
|
||||
encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_failures_total",
|
||||
Help: "Total number of failed automatic reloads of encryption configuration.",
|
||||
Help: "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"apiserver_id_hash"},
|
||||
)
|
||||
|
||||
encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter(
|
||||
encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_success_total",
|
||||
Help: "Total number of successful automatic reloads of encryption configuration.",
|
||||
Help: "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"apiserver_id_hash"},
|
||||
)
|
||||
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
|
||||
@ -54,33 +59,53 @@ var (
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_last_timestamp_seconds",
|
||||
Help: "Timestamp of the last successful or failed automatic reload of encryption configuration.",
|
||||
Help: "Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"status"},
|
||||
[]string{"status", "apiserver_id_hash"},
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
var hashPool *sync.Pool
|
||||
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
hashPool = &sync.Pool{
|
||||
New: func() interface{} {
|
||||
return sha256.New()
|
||||
},
|
||||
}
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
|
||||
})
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadFailure() {
|
||||
encryptionConfigAutomaticReloadFailureTotal.Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("failure")
|
||||
func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
|
||||
apiServerIDHash := getHash(apiServerID)
|
||||
encryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadSuccess() {
|
||||
encryptionConfigAutomaticReloadSuccessTotal.Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("success")
|
||||
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
|
||||
apiServerIDHash := getHash(apiServerID)
|
||||
encryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
|
||||
}
|
||||
|
||||
func recordEncryptionConfigAutomaticReloadTimestamp(result string) {
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime()
|
||||
func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
|
||||
}
|
||||
|
||||
func getHash(data string) string {
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
}
|
||||
h := hashPool.Get().(hash.Hash)
|
||||
h.Reset()
|
||||
h.Write([]byte(data))
|
||||
dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
|
||||
hashPool.Put(h)
|
||||
return dataHash
|
||||
}
|
||||
|
||||
@ -24,11 +24,16 @@ import (
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
testAPIServerID = "testAPIServerID"
|
||||
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
|
||||
)
|
||||
|
||||
func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
|
||||
expectedValue := `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration.
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter
|
||||
apiserver_encryption_config_controller_automatic_reload_failures_total 1
|
||||
apiserver_encryption_config_controller_automatic_reload_failures_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"} 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reload_failures_total",
|
||||
@ -37,7 +42,7 @@ func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
|
||||
encryptionConfigAutomaticReloadFailureTotal.Reset()
|
||||
RegisterMetrics()
|
||||
|
||||
RecordEncryptionConfigAutomaticReloadFailure()
|
||||
RecordEncryptionConfigAutomaticReloadFailure(testAPIServerID)
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -45,9 +50,9 @@ func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
|
||||
|
||||
func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
|
||||
expectedValue := `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration.
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter
|
||||
apiserver_encryption_config_controller_automatic_reload_success_total 1
|
||||
apiserver_encryption_config_controller_automatic_reload_success_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"} 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reload_success_total",
|
||||
@ -56,7 +61,7 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
|
||||
encryptionConfigAutomaticReloadSuccessTotal.Reset()
|
||||
RegisterMetrics()
|
||||
|
||||
RecordEncryptionConfigAutomaticReloadSuccess()
|
||||
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID)
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -70,18 +75,18 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
expectedValue: `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration.
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
|
||||
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="failure"} 1.689101941e+09
|
||||
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="failure"} 1.689101941e+09
|
||||
`,
|
||||
resultLabel: "failure",
|
||||
timestamp: 1689101941,
|
||||
},
|
||||
{
|
||||
expectedValue: `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration.
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
|
||||
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="success"} 1.689101941e+09
|
||||
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1.689101941e+09
|
||||
`,
|
||||
resultLabel: "success",
|
||||
timestamp: 1689101941,
|
||||
@ -95,7 +100,7 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel).Set(float64(tc.timestamp))
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel, testAPIServerIDHash).Set(float64(tc.timestamp))
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@ -294,7 +294,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
|
||||
}
|
||||
}()
|
||||
|
||||
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload)
|
||||
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -318,6 +318,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
|
||||
s.EncryptionProviderConfigFilepath,
|
||||
dynamicTransformers,
|
||||
encryptionConfiguration.EncryptionFileContentHash,
|
||||
c.APIServerID,
|
||||
)
|
||||
|
||||
go dynamicEncryptionConfigController.Run(ctxServer)
|
||||
|
||||
@ -110,22 +110,24 @@ type envelopeTransformer struct {
|
||||
stateFunc StateFunc
|
||||
|
||||
// cache is a thread-safe expiring lru cache which caches decrypted DEKs indexed by their encrypted form.
|
||||
cache *simpleCache
|
||||
cache *simpleCache
|
||||
apiServerID string
|
||||
}
|
||||
|
||||
// NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme.
|
||||
// It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
|
||||
// the data items they encrypt.
|
||||
func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc) value.Transformer {
|
||||
return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, cacheTTL, clock.RealClock{})
|
||||
func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string) value.Transformer {
|
||||
return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, apiServerID, cacheTTL, clock.RealClock{})
|
||||
}
|
||||
|
||||
func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, cacheTTL time.Duration, clock clock.Clock) value.Transformer {
|
||||
func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string, cacheTTL time.Duration, clock clock.Clock) value.Transformer {
|
||||
return &envelopeTransformer{
|
||||
envelopeService: envelopeService,
|
||||
providerName: providerName,
|
||||
stateFunc: stateFunc,
|
||||
cache: newSimpleCache(clock, cacheTTL, providerName),
|
||||
apiServerID: apiServerID,
|
||||
}
|
||||
}
|
||||
|
||||
@ -178,7 +180,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID)
|
||||
metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID, t.apiServerID)
|
||||
|
||||
out, stale, err := transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx)
|
||||
if err != nil {
|
||||
@ -218,7 +220,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID)
|
||||
metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID, t.apiServerID)
|
||||
|
||||
encObjectCopy := state.EncryptedObject
|
||||
encObjectCopy.EncryptedData = result
|
||||
|
||||
@ -49,10 +49,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
testText = "abcdefghijklmnopqrstuvwxyz"
|
||||
testContextText = "0123456789"
|
||||
testKeyHash = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
|
||||
testKeyVersion = "1"
|
||||
testText = "abcdefghijklmnopqrstuvwxyz"
|
||||
testContextText = "0123456789"
|
||||
testKeyHash = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
|
||||
testKeyVersion = "1"
|
||||
testAPIServerID = "testAPIServerID"
|
||||
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
|
||||
)
|
||||
|
||||
// testEnvelopeService is a mock Envelope service which can be used to simulate remote Envelope services
|
||||
@ -178,7 +180,7 @@ func TestEnvelopeCaching(t *testing.T) {
|
||||
}
|
||||
|
||||
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
|
||||
func() (State, error) { return state, nil },
|
||||
func() (State, error) { return state, nil }, testAPIServerID,
|
||||
tt.cacheTTL, fakeClock)
|
||||
|
||||
dataCtx := value.DefaultContext(testContextText)
|
||||
@ -319,7 +321,7 @@ func TestEnvelopeTransformerStaleness(t *testing.T) {
|
||||
var stateErr error
|
||||
|
||||
transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
|
||||
func() (State, error) { return state, stateErr },
|
||||
func() (State, error) { return state, stateErr }, testAPIServerID,
|
||||
)
|
||||
|
||||
dataCtx := value.DefaultContext(testContextText)
|
||||
@ -376,7 +378,7 @@ func TestEnvelopeTransformerStateFunc(t *testing.T) {
|
||||
stateErr := fmt.Errorf("some state error")
|
||||
|
||||
transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
|
||||
func() (State, error) { return state, stateErr },
|
||||
func() (State, error) { return state, stateErr }, testAPIServerID,
|
||||
)
|
||||
|
||||
dataCtx := value.DefaultContext(testContextText)
|
||||
@ -513,6 +515,7 @@ func TestTransformToStorageError(t *testing.T) {
|
||||
envelopeService.SetAnnotations(tt.annotations)
|
||||
transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
|
||||
testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool()),
|
||||
testAPIServerID,
|
||||
)
|
||||
dataCtx := value.DefaultContext(testContextText)
|
||||
|
||||
@ -838,6 +841,7 @@ func TestEnvelopeMetrics(t *testing.T) {
|
||||
envelopeService := newTestEnvelopeService()
|
||||
transformer := NewEnvelopeTransformer(envelopeService, testProviderName,
|
||||
testStateFunc(testContext(t), envelopeService, clock.RealClock{}, randomBool()),
|
||||
testAPIServerID,
|
||||
)
|
||||
|
||||
dataCtx := value.DefaultContext(testContextText)
|
||||
@ -859,11 +863,11 @@ func TestEnvelopeMetrics(t *testing.T) {
|
||||
"apiserver_envelope_encryption_key_id_hash_total",
|
||||
},
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider.
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
|
||||
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testKeyHash, testProviderName, metrics.FromStorageLabel, testKeyHash, testProviderName, metrics.ToStorageLabel),
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testAPIServerIDHash, testKeyHash, testProviderName, metrics.FromStorageLabel, testAPIServerIDHash, testKeyHash, testProviderName, metrics.ToStorageLabel),
|
||||
},
|
||||
}
|
||||
|
||||
@ -931,10 +935,10 @@ func TestEnvelopeMetricsCache(t *testing.T) {
|
||||
transformer1 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) {
|
||||
// return different states to ensure we get expected number of cache keys after restart on decryption
|
||||
return testStateFunc(ctx, envelopeService, clock.RealClock{}, randomBool())()
|
||||
})
|
||||
transformer2 := NewEnvelopeTransformer(envelopeService, provider2, func() (State, error) { return state, nil })
|
||||
}, testAPIServerID)
|
||||
transformer2 := NewEnvelopeTransformer(envelopeService, provider2, func() (State, error) { return state, nil }, testAPIServerID)
|
||||
// used for restart
|
||||
transformer3 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { return state, nil })
|
||||
transformer3 := NewEnvelopeTransformer(envelopeService, provider1, func() (State, error) { return state, nil }, testAPIServerID)
|
||||
var transformedDatas [][]byte
|
||||
for j := 0; j < numOfStates; j++ {
|
||||
transformedData, err := transformer1.TransformToStorage(ctx, []byte(testText), dataCtx)
|
||||
@ -1029,8 +1033,7 @@ func TestEnvelopeLogging(t *testing.T) {
|
||||
envelopeService := newTestEnvelopeService()
|
||||
fakeClock := testingclock.NewFakeClock(time.Now())
|
||||
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
|
||||
testStateFunc(tc.ctx, envelopeService, clock.RealClock{}, randomBool()),
|
||||
1*time.Second, fakeClock)
|
||||
testStateFunc(tc.ctx, envelopeService, clock.RealClock{}, randomBool()), testAPIServerID, 1*time.Second, fakeClock)
|
||||
|
||||
dataCtx := value.DefaultContext([]byte(testContextText))
|
||||
originalText := []byte(testText)
|
||||
@ -1082,7 +1085,7 @@ func TestCacheNotCorrupted(t *testing.T) {
|
||||
}
|
||||
|
||||
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
|
||||
func() (State, error) { return state, nil },
|
||||
func() (State, error) { return state, nil }, testAPIServerID,
|
||||
1*time.Second, fakeClock)
|
||||
|
||||
dataCtx := value.DefaultContext(testContextText)
|
||||
@ -1108,7 +1111,7 @@ func TestCacheNotCorrupted(t *testing.T) {
|
||||
}
|
||||
|
||||
transformer = newEnvelopeTransformerWithClock(envelopeService, testProviderName,
|
||||
func() (State, error) { return state, nil },
|
||||
func() (State, error) { return state, nil }, testAPIServerID,
|
||||
1*time.Second, fakeClock)
|
||||
|
||||
transformedData2, err := transformer.TransformToStorage(ctx, originalText, dataCtx)
|
||||
|
||||
@ -44,6 +44,7 @@ type metricLabels struct {
|
||||
transformationType string
|
||||
providerName string
|
||||
keyIDHash string
|
||||
apiServerIDHash string
|
||||
}
|
||||
|
||||
/*
|
||||
@ -107,21 +108,21 @@ var (
|
||||
|
||||
// keyIDHashTotal is the number of times a keyID is used
|
||||
// e.g. apiserver_envelope_encryption_key_id_hash_total counter
|
||||
// apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256",
|
||||
// apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256",key_id_hash="sha256",
|
||||
// provider_name="providerName",transformation_type="from_storage"} 1
|
||||
KeyIDHashTotal = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "key_id_hash_total",
|
||||
Help: "Number of times a keyID is used split by transformation type and provider.",
|
||||
Help: "Number of times a keyID is used split by transformation type, provider, and apiserver identity.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"transformation_type", "provider_name", "key_id_hash"},
|
||||
[]string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"},
|
||||
)
|
||||
|
||||
// keyIDHashLastTimestampSeconds is the last time in seconds when a keyID was used
|
||||
// e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09
|
||||
// e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09
|
||||
KeyIDHashLastTimestampSeconds = metrics.NewGaugeVec(
|
||||
&metrics.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
@ -130,11 +131,11 @@ var (
|
||||
Help: "The last time in seconds when a keyID was used.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"transformation_type", "provider_name", "key_id_hash"},
|
||||
[]string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"},
|
||||
)
|
||||
|
||||
// keyIDHashStatusLastTimestampSeconds is the last time in seconds when a keyID was returned by the Status RPC call.
|
||||
// e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09
|
||||
// e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09
|
||||
KeyIDHashStatusLastTimestampSeconds = metrics.NewGaugeVec(
|
||||
&metrics.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
@ -143,7 +144,7 @@ var (
|
||||
Help: "The last time in seconds when a keyID was returned by the Status RPC call.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"provider_name", "key_id_hash"},
|
||||
[]string{"provider_name", "key_id_hash", "apiserver_id_hash"},
|
||||
)
|
||||
|
||||
InvalidKeyIDFromStatusTotal = metrics.NewCounterVec(
|
||||
@ -182,19 +183,19 @@ func registerLRUMetrics() {
|
||||
|
||||
keyIDHashTotalMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
|
||||
item := key.(metricLabels)
|
||||
if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted {
|
||||
if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
|
||||
klog.InfoS("Deleted keyIDHashTotalMetricLabels", "transformationType", item.transformationType,
|
||||
"providerName", item.providerName, "keyIDHash", item.keyIDHash)
|
||||
"providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
|
||||
}
|
||||
if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted {
|
||||
if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
|
||||
klog.InfoS("Deleted keyIDHashLastTimestampSecondsMetricLabels", "transformationType", item.transformationType,
|
||||
"providerName", item.providerName, "keyIDHash", item.keyIDHash)
|
||||
"providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
|
||||
}
|
||||
})
|
||||
keyIDHashStatusLastTimestampSecondsMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
|
||||
item := key.(metricLabels)
|
||||
if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash); deleted {
|
||||
klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash)
|
||||
if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
|
||||
klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -218,22 +219,22 @@ func RegisterMetrics() {
|
||||
}
|
||||
|
||||
// RecordKeyID records total count and last time in seconds when a KeyID was used for TransformFromStorage and TransformToStorage operations
|
||||
func RecordKeyID(transformationType, providerName, keyID string) {
|
||||
func RecordKeyID(transformationType, providerName, keyID, apiServerID string) {
|
||||
lockRecordKeyID.Lock()
|
||||
defer lockRecordKeyID.Unlock()
|
||||
|
||||
keyIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID)
|
||||
KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash).Inc()
|
||||
KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash).SetToCurrentTime()
|
||||
keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID, apiServerID)
|
||||
KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).Inc()
|
||||
KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).SetToCurrentTime()
|
||||
}
|
||||
|
||||
// RecordKeyIDFromStatus records last time in seconds when a KeyID was returned by the Status RPC call.
|
||||
func RecordKeyIDFromStatus(providerName, keyID string) {
|
||||
func RecordKeyIDFromStatus(providerName, keyID, apiServerID string) {
|
||||
lockRecordKeyIDStatus.Lock()
|
||||
defer lockRecordKeyIDStatus.Unlock()
|
||||
|
||||
keyIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID)
|
||||
KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash).SetToCurrentTime()
|
||||
keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID, apiServerID)
|
||||
KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash, apiServerIDHash).SetToCurrentTime()
|
||||
}
|
||||
|
||||
func RecordInvalidKeyIDFromStatus(providerName, errCode string) {
|
||||
@ -297,24 +298,25 @@ func getErrorCode(err error) string {
|
||||
}
|
||||
|
||||
func getHash(data string) string {
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
}
|
||||
h := hashPool.Get().(hash.Hash)
|
||||
h.Reset()
|
||||
h.Write([]byte(data))
|
||||
result := fmt.Sprintf("sha256:%x", h.Sum(nil))
|
||||
dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
|
||||
hashPool.Put(h)
|
||||
return result
|
||||
return dataHash
|
||||
}
|
||||
|
||||
func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID string) string {
|
||||
keyIDHash := ""
|
||||
// only get hash if the keyID is not empty
|
||||
if len(keyID) > 0 {
|
||||
keyIDHash = getHash(keyID)
|
||||
}
|
||||
func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID, apiServerID string) (string, string) {
|
||||
keyIDHash := getHash(keyID)
|
||||
apiServerIDHash := getHash(apiServerID)
|
||||
c.Add(metricLabels{
|
||||
transformationType: transformationType,
|
||||
providerName: providerName,
|
||||
keyIDHash: keyIDHash,
|
||||
apiServerIDHash: apiServerIDHash,
|
||||
}, nil) // value is irrelevant, this is a set and not a map
|
||||
return keyIDHash
|
||||
return keyIDHash, apiServerIDHash
|
||||
}
|
||||
|
||||
@ -36,6 +36,8 @@ const (
|
||||
testKeyHash2 = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35"
|
||||
testKeyHash3 = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce"
|
||||
testProviderNameForMetric = "providerName"
|
||||
testAPIServerID = "testAPIServerID"
|
||||
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -204,6 +206,7 @@ func TestRecordKeyID_Serial(t *testing.T) {
|
||||
metrics []string
|
||||
providerName string
|
||||
transformationType string
|
||||
apiServerID string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
@ -214,11 +217,12 @@ func TestRecordKeyID_Serial(t *testing.T) {
|
||||
},
|
||||
providerName: testProviderNameForMetric,
|
||||
transformationType: FromStorageLabel,
|
||||
apiServerID: testAPIServerID,
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider.
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
|
||||
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testKeyHash1, testProviderNameForMetric, FromStorageLabel),
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel),
|
||||
},
|
||||
{
|
||||
desc: "keyIDHash total more labels",
|
||||
@ -228,12 +232,13 @@ func TestRecordKeyID_Serial(t *testing.T) {
|
||||
},
|
||||
providerName: testProviderNameForMetric,
|
||||
transformationType: FromStorageLabel,
|
||||
apiServerID: testAPIServerID,
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider.
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
|
||||
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testKeyHash2, testProviderNameForMetric, FromStorageLabel),
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel),
|
||||
},
|
||||
{
|
||||
desc: "keyIDHash total same labels",
|
||||
@ -243,12 +248,13 @@ func TestRecordKeyID_Serial(t *testing.T) {
|
||||
},
|
||||
providerName: testProviderNameForMetric,
|
||||
transformationType: FromStorageLabel,
|
||||
apiServerID: testAPIServerID,
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider.
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
|
||||
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2
|
||||
`, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testKeyHash2, testProviderNameForMetric, FromStorageLabel),
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2
|
||||
`, testAPIServerIDHash, testKeyHash1, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel),
|
||||
},
|
||||
{
|
||||
desc: "keyIDHash total exceeds limit, remove first label, and empty keyID",
|
||||
@ -258,12 +264,29 @@ func TestRecordKeyID_Serial(t *testing.T) {
|
||||
},
|
||||
providerName: testProviderNameForMetric,
|
||||
transformationType: FromStorageLabel,
|
||||
apiServerID: testAPIServerID,
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider.
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
|
||||
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testKeyHash2, testProviderNameForMetric, FromStorageLabel, "", testProviderNameForMetric, FromStorageLabel),
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 2
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testAPIServerIDHash, testKeyHash2, testProviderNameForMetric, FromStorageLabel, testAPIServerIDHash, "", testProviderNameForMetric, FromStorageLabel),
|
||||
},
|
||||
{
|
||||
desc: "keyIDHash total exceeds limit, remove first label, empty keyID, and empty testAPIServerID",
|
||||
keyID: "",
|
||||
metrics: []string{
|
||||
"apiserver_envelope_encryption_key_id_hash_total",
|
||||
},
|
||||
providerName: testProviderNameForMetric,
|
||||
transformationType: FromStorageLabel,
|
||||
apiServerID: "",
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
|
||||
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, testAPIServerIDHash, "", testProviderNameForMetric, FromStorageLabel, "", "", testProviderNameForMetric, FromStorageLabel),
|
||||
},
|
||||
{
|
||||
desc: "keyIDHash total exceeds limit 2, remove first label",
|
||||
@ -273,12 +296,13 @@ func TestRecordKeyID_Serial(t *testing.T) {
|
||||
},
|
||||
providerName: testProviderNameForMetric,
|
||||
transformationType: FromStorageLabel,
|
||||
apiServerID: "",
|
||||
want: fmt.Sprintf(`
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type and provider.
|
||||
# HELP apiserver_envelope_encryption_key_id_hash_total [ALPHA] Number of times a keyID is used split by transformation type, provider, and apiserver identity.
|
||||
# TYPE apiserver_envelope_encryption_key_id_hash_total counter
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, "", testProviderNameForMetric, FromStorageLabel, testKeyHash1, testProviderNameForMetric, FromStorageLabel),
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="%s",key_id_hash="%s",provider_name="%s",transformation_type="%s"} 1
|
||||
`, "", "", testProviderNameForMetric, FromStorageLabel, "", testKeyHash1, testProviderNameForMetric, FromStorageLabel),
|
||||
},
|
||||
}
|
||||
|
||||
@ -289,7 +313,7 @@ func TestRecordKeyID_Serial(t *testing.T) {
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.desc, func(t *testing.T) {
|
||||
RecordKeyID(tt.transformationType, tt.providerName, tt.keyID)
|
||||
RecordKeyID(tt.transformationType, tt.providerName, tt.keyID, tt.apiServerID)
|
||||
// We are not resetting the metric here as each test is not independent in order to validate the behavior
|
||||
// when the metric labels exceed the limit to ensure the labels are not unbounded.
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
|
||||
@ -313,12 +337,14 @@ func TestRecordKeyIDLRUKey(t *testing.T) {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
keyID := rand.String(32)
|
||||
apiServerID := rand.String(32)
|
||||
key := metricLabels{
|
||||
transformationType: rand.String(32),
|
||||
providerName: rand.String(32),
|
||||
keyIDHash: getHash(keyID),
|
||||
apiServerIDHash: getHash(apiServerID),
|
||||
}
|
||||
RecordKeyID(key.transformationType, key.providerName, keyID)
|
||||
RecordKeyID(key.transformationType, key.providerName, keyID, apiServerID)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
@ -359,11 +385,13 @@ func TestRecordKeyIDFromStatus(t *testing.T) {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
keyID := rand.String(32)
|
||||
apiServerID := rand.String(32)
|
||||
key := metricLabels{
|
||||
providerName: rand.String(32),
|
||||
keyIDHash: getHash(keyID),
|
||||
providerName: rand.String(32),
|
||||
keyIDHash: getHash(keyID),
|
||||
apiServerIDHash: getHash(apiServerID),
|
||||
}
|
||||
RecordKeyIDFromStatus(key.providerName, keyID)
|
||||
RecordKeyIDFromStatus(key.providerName, keyID, apiServerID)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
@ -29,21 +29,27 @@ import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"golang.org/x/crypto/cryptobyte"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
"k8s.io/apiserver/pkg/storage/value"
|
||||
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
|
||||
mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1"
|
||||
"k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/client-go/rest"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
kmsapi "k8s.io/kms/apis/v1beta1"
|
||||
"k8s.io/kubernetes/test/integration"
|
||||
@ -308,6 +314,8 @@ resources:
|
||||
cachesize: 1000
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
`
|
||||
|
||||
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
|
||||
_ = mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
var restarted bool
|
||||
test, err := newTransformTest(t, encryptionConfig, true, "", storageConfig)
|
||||
@ -319,6 +327,26 @@ resources:
|
||||
test.cleanUp()
|
||||
}
|
||||
}()
|
||||
ctx := testContext(t)
|
||||
|
||||
// the global metrics registry persists across test runs - reset it here so we can make assertions
|
||||
copyConfig := rest.CopyConfig(test.kubeAPIServer.ClientConfig)
|
||||
copyConfig.GroupVersion = &schema.GroupVersion{}
|
||||
copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
|
||||
rc, err := rest.RESTClientFor(copyConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rc.Delete().AbsPath("/metrics").Do(ctx).Error(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// assert that the metrics we collect during the test run match expectations
|
||||
// NOTE: 2 successful automatic reload resulted from 2 config file updates
|
||||
wantMetricStrings := []string{
|
||||
`apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_encryption_config_controller_automatic_reload_success_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795"} 2`,
|
||||
}
|
||||
|
||||
test.secret, err = test.createSecret(testSecret, testNamespace)
|
||||
if err != nil {
|
||||
@ -378,7 +406,7 @@ resources:
|
||||
|
||||
// run storage migration
|
||||
// get secrets
|
||||
ctx := testContext(t)
|
||||
|
||||
secretsList, err := test.restClient.CoreV1().Secrets("").List(
|
||||
ctx,
|
||||
metav1.ListOptions{},
|
||||
@ -522,6 +550,34 @@ resources:
|
||||
if _, err = test.restClient.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{}); err != nil {
|
||||
t.Fatalf("failed to list configmaps, err: %v", err)
|
||||
}
|
||||
|
||||
// recreate rest client with the new transformTest
|
||||
copyConfig = rest.CopyConfig(test.kubeAPIServer.ClientConfig)
|
||||
copyConfig.GroupVersion = &schema.GroupVersion{}
|
||||
copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
|
||||
rc, err = rest.RESTClientFor(copyConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var gotMetricStrings []string
|
||||
trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`)
|
||||
for _, line := range strings.Split(string(body), "\n") {
|
||||
if strings.HasPrefix(line, "apiserver_encryption_config_controller_") {
|
||||
if strings.Contains(line, "_seconds") {
|
||||
line = trimFP.ReplaceAllString(line, `$1`) + "} FP" // ignore floating point metric values
|
||||
}
|
||||
gotMetricStrings = append(gotMetricStrings, line)
|
||||
}
|
||||
}
|
||||
if diff := cmp.Diff(wantMetricStrings, gotMetricStrings); diff != "" {
|
||||
t.Errorf("unexpected metrics diff (-want +got): %s", diff)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func TestEncryptAll(t *testing.T) {
|
||||
|
||||
@ -199,7 +199,7 @@ resources:
|
||||
name: kms-provider
|
||||
endpoint: unix:///@kms-provider.sock
|
||||
`
|
||||
|
||||
genericapiserver.SetHostnameFuncForTests("testAPIServerID")
|
||||
providerName := "kms-provider"
|
||||
pluginMock := kmsv2mock.NewBase64Plugin(t, "@kms-provider.sock")
|
||||
|
||||
@ -226,10 +226,10 @@ resources:
|
||||
// assert that the metrics we collect during the test run match expectations
|
||||
wantMetricStrings := []string{
|
||||
`apiserver_envelope_encryption_dek_source_cache_size{provider_name="kms-provider"} 1`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} FP`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="from_storage"} 2`,
|
||||
`apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",key_id_hash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",provider_name="kms-provider",transformation_type="to_storage"} 1`,
|
||||
}
|
||||
defer func() {
|
||||
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user