diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index 9fb9c4776b7..374f6e3d410 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -83,14 +83,17 @@ func createAPIExtensionsConfig( apiextensionsapiserver.Scheme); err != nil { return nil, err } - + crdRESTOptionsGetter, err := apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions) + if err != nil { + return nil, err + } apiextensionsConfig := &apiextensionsapiserver.Config{ GenericConfig: &genericapiserver.RecommendedConfig{ Config: genericConfig, SharedInformerFactory: externalInformers, }, ExtraConfig: apiextensionsapiserver.ExtraConfig{ - CRDRESTOptionsGetter: apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions), + CRDRESTOptionsGetter: crdRESTOptionsGetter, MasterCount: masterCount, AuthResolverWrapper: authResolverWrapper, ServiceResolver: serviceResolver, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index d646aa992ca..dd654ef1a0c 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -70,10 +70,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/metrics" apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic" - genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" genericfilters "k8s.io/apiserver/pkg/server/filters" - "k8s.io/apiserver/pkg/storage/storagebackend" - flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" utilopenapi "k8s.io/apiserver/pkg/util/openapi" "k8s.io/apiserver/pkg/util/webhook" "k8s.io/apiserver/pkg/warning" @@ -1134,33 +1131,6 @@ func (d unstructuredDefaulter) Default(in runtime.Object) { structuraldefaulting.Default(u.UnstructuredContent(), d.structuralSchemas[u.GetObjectKind().GroupVersionKind().Version]) } -type CRDRESTOptionsGetter struct { - StorageConfig storagebackend.Config - StoragePrefix string - EnableWatchCache bool - DefaultWatchCacheSize int - EnableGarbageCollection bool - DeleteCollectionWorkers int - CountMetricPollPeriod time.Duration - StorageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker -} - -func (t CRDRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - ret := generic.RESTOptions{ - StorageConfig: t.StorageConfig.ForResource(resource), - Decorator: generic.UndecoratedStorage, - EnableGarbageCollection: t.EnableGarbageCollection, - DeleteCollectionWorkers: t.DeleteCollectionWorkers, - ResourcePrefix: resource.Group + "/" + resource.Resource, - CountMetricPollPeriod: t.CountMetricPollPeriod, - StorageObjectCountTracker: t.StorageObjectCountTracker, - } - if t.EnableWatchCache { - ret.Decorator = genericregistry.StorageWithCacher() - } - return ret, nil -} - // clone returns a clone of the provided crdStorageMap. // The clone is a shallow copy of the map. func (in crdStorageMap) clone() crdStorageMap { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go index 97a662be321..158b6e13844 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go @@ -21,6 +21,7 @@ import ( "io" "net" "net/url" + "reflect" "github.com/spf13/pflag" oteltrace "go.opentelemetry.io/otel/trace" @@ -106,11 +107,14 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err if err := o.APIEnablement.ApplyTo(&serverConfig.Config, apiserver.DefaultAPIResourceConfigSource(), apiserver.Scheme); err != nil { return nil, err } - + crdRESTOptionsGetter, err := NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd) + if err != nil { + return nil, err + } config := &apiserver.Config{ GenericConfig: serverConfig, ExtraConfig: apiserver.ExtraConfig{ - CRDRESTOptionsGetter: NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd), + CRDRESTOptionsGetter: crdRESTOptionsGetter, ServiceResolver: &serviceResolver{serverConfig.SharedInformerFactory.Core().V1().Services().Lister()}, AuthResolverWrapper: webhook.NewDefaultAuthenticationInfoResolverWrapper(nil, nil, serverConfig.LoopbackClientConfig, oteltrace.NewNoopTracerProvider()), }, @@ -119,20 +123,30 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err } // NewCRDRESTOptionsGetter create a RESTOptionsGetter for CustomResources. -func NewCRDRESTOptionsGetter(etcdOptions genericoptions.EtcdOptions) genericregistry.RESTOptionsGetter { - ret := apiserver.CRDRESTOptionsGetter{ - StorageConfig: etcdOptions.StorageConfig, - StoragePrefix: etcdOptions.StorageConfig.Prefix, - EnableWatchCache: etcdOptions.EnableWatchCache, - DefaultWatchCacheSize: etcdOptions.DefaultWatchCacheSize, - EnableGarbageCollection: etcdOptions.EnableGarbageCollection, - DeleteCollectionWorkers: etcdOptions.DeleteCollectionWorkers, - CountMetricPollPeriod: etcdOptions.StorageConfig.CountMetricPollPeriod, - StorageObjectCountTracker: etcdOptions.StorageConfig.StorageObjectCountTracker, - } - ret.StorageConfig.Codec = unstructured.UnstructuredJSONScheme +// This works on a copy of the etcd options so we don't mutate originals. +// We assume that the input etcd options have been completed already. +// Avoid messing with anything outside of changes to StorageConfig as that +// may lead to unexpected behavior when the options are applied. +func NewCRDRESTOptionsGetter(etcdOptions genericoptions.EtcdOptions) (genericregistry.RESTOptionsGetter, error) { + etcdOptions.StorageConfig.Codec = unstructured.UnstructuredJSONScheme + etcdOptions.WatchCacheSizes = nil // this control is not provided for custom resources + etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks - return ret + // creates a generic apiserver config for etcdOptions to mutate + c := genericapiserver.Config{} + if err := etcdOptions.ApplyTo(&c); err != nil { + return nil, err + } + restOptionsGetter := c.RESTOptionsGetter + if restOptionsGetter == nil { + return nil, fmt.Errorf("server.Config RESTOptionsGetter should not be nil") + } + // sanity check that no other fields are set + c.RESTOptionsGetter = nil + if !reflect.DeepEqual(c, genericapiserver.Config{}) { + return nil, fmt.Errorf("only RESTOptionsGetter should have been mutated in server.Config") + } + return restOptionsGetter, nil } type serviceResolver struct { diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go index 43a0a3dc451..c93cbe99cc5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go @@ -182,7 +182,10 @@ func testWebhookConverter(t *testing.T, watchCache bool) { crd := multiVersionFixture.DeepCopy() - RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + RESTOptionsGetter, err := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + if err != nil { + t.Fatal(err) + } restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) if err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go index 351f4cc453c..cffe99072ec 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go @@ -658,7 +658,10 @@ func TestCustomResourceDefaultingOfMetaFields(t *testing.T) { t.Logf("CR created: %#v", returnedFoo.UnstructuredContent()) // get persisted object - RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + RESTOptionsGetter, err := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + if err != nil { + t.Fatal(err) + } restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) if err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go index f330e0d65e5..72fba20d0c9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go @@ -144,7 +144,10 @@ func StartDefaultServerWithClientsAndEtcd(t servertesting.Logger, extraFlags ... return nil, nil, nil, nil, "", err } - RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + RESTOptionsGetter, err := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + if err != nil { + return nil, nil, nil, nil, "", err + } restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"}) if err != nil { return nil, nil, nil, nil, "", err diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go index fe6f0607cb2..319fc9fbf52 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go @@ -132,7 +132,10 @@ func TestInvalidObjectMetaInStorage(t *testing.T) { t.Fatal(err) } - RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + RESTOptionsGetter, err := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + if err != nil { + t.Fatal(err) + } restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural}) if err != nil { t.Fatal(err) diff --git a/test/integration/controlplane/transformation/all_transformation_test.go b/test/integration/controlplane/transformation/all_transformation_test.go new file mode 100644 index 00000000000..bf6775df6b0 --- /dev/null +++ b/test/integration/controlplane/transformation/all_transformation_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformation + +import ( + "context" + "testing" + "time" + + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/kubernetes/test/integration/etcd" +) + +func createResources(t *testing.T, test *transformTest, + group, + version, + kind, + resource, + name, + namespace string, +) { + switch resource { + case "pods": + _, err := test.createPod(namespace, dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)) + if err != nil { + t.Fatalf("Failed to create test pod, error: %v, name: %s, ns: %s", err, name, namespace) + } + case "configmaps": + _, err := test.createConfigMap(name, namespace) + if err != nil { + t.Fatalf("Failed to create test configmap, error: %v, name: %s, ns: %s", err, name, namespace) + } + default: + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + + gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: resource} + data := etcd.GetEtcdStorageData()[gvr] + stub := data.Stub + dynamicClient, obj, err := etcd.JSONToUnstructured(stub, namespace, &meta.RESTMapping{ + Resource: gvr, + GroupVersionKind: gvr.GroupVersion().WithKind(kind), + Scope: meta.RESTScopeRoot, + }, dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)) + if err != nil { + t.Fatal(err) + } + _, err = dynamicClient.Create(ctx, obj, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := dynamicClient.Get(ctx, obj.GetName(), metav1.GetOptions{}); err != nil { + t.Fatalf("object should exist: %v", err) + } + } +} + +func TestEncryptSupportedForAllResourceTypes(t *testing.T) { + // check resources provided by the three servers that we have wired together + // - pods and configmaps from KAS + // - CRDs and CRs from API extensions + // - API services from aggregator + encryptionConfig := ` +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: +- resources: + - pods + - configmaps + - customresourcedefinitions.apiextensions.k8s.io + - pandas.awesome.bears.com + - apiservices.apiregistration.k8s.io + providers: + - aescbc: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== +` + + test, err := newTransformTest(t, encryptionConfig) + if err != nil { + t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) + } + t.Cleanup(test.cleanUp) + + // the storage registry for CRs is dynamic so create one to exercise the wiring + etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(test.kubeAPIServer.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...) + + for _, tt := range []struct { + group string + version string + kind string + resource string + name string + namespace string + }{ + {"", "v1", "ConfigMap", "configmaps", "cm1", testNamespace}, + {"apiextensions.k8s.io", "v1", "CustomResourceDefinition", "customresourcedefinitions", "pandas.awesome.bears.com", ""}, + {"awesome.bears.com", "v1", "Panda", "pandas", "cr3panda", ""}, + {"apiregistration.k8s.io", "v1", "APIService", "apiservices", "as2.foo.com", ""}, + {"", "v1", "Pod", "pods", "pod1", testNamespace}, + } { + tt := tt + t.Run(tt.resource, func(t *testing.T) { + t.Parallel() + + createResources(t, test, tt.group, tt.version, tt.kind, tt.resource, tt.name, tt.namespace) + test.runResource(t, unSealWithCBCTransformer, aesCBCPrefix, tt.group, tt.version, tt.resource, tt.name, tt.namespace) + }) + } +} diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 9773296ee29..c0b5342001d 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -145,7 +145,7 @@ resources: // Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it. plainTextDEK := pluginMock.LastEncryptRequest() - secretETCDPath := test.getETCDPath() + secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace) rawEnvelope, err := test.getRawSecretFromETCD() if err != nil { t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err) diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index dfb250a938f..1865dbd2cb7 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -154,7 +154,7 @@ resources: // Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it. plainTextDEK := pluginMock.LastEncryptRequest() - secretETCDPath := test.getETCDPath() + secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace) rawEnvelope, err := test.getRawSecretFromETCD() if err != nil { t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err) diff --git a/test/integration/controlplane/transformation/secrets_transformation_test.go b/test/integration/controlplane/transformation/secrets_transformation_test.go index b899237a937..ccbd86ef0ff 100644 --- a/test/integration/controlplane/transformation/secrets_transformation_test.go +++ b/test/integration/controlplane/transformation/secrets_transformation_test.go @@ -95,7 +95,7 @@ func TestSecretsShouldBeTransformed(t *testing.T) { if err != nil { t.Fatalf("Failed to create test secret, error: %v", err) } - test.run(tt.unSealFunc, tt.transformerPrefix) + test.runResource(test.logger, tt.unSealFunc, tt.transformerPrefix, "", "v1", "secrets", test.secret.Name, test.secret.Namespace) test.cleanUp() } } diff --git a/test/integration/controlplane/transformation/transformation_testcase.go b/test/integration/controlplane/transformation/transformation_testcase.go index 7fde0cb628c..14da3360eba 100644 --- a/test/integration/controlplane/transformation/transformation_testcase.go +++ b/test/integration/controlplane/transformation/transformation_testcase.go @@ -19,6 +19,7 @@ package transformation import ( "bytes" "context" + "encoding/json" "fmt" "os" "path" @@ -34,12 +35,16 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storage/value" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration" + "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" ) @@ -50,6 +55,8 @@ const ( testNamespace = "secret-encryption-test" testSecret = "test-secret" metricsPrefix = "apiserver_storage_" + configMapKey = "foo" + configMapVal = "bar" // precomputed key and secret for use with AES CBC // this looks exactly the same as the AES GCM secret but with a different value @@ -107,44 +114,88 @@ func (e *transformTest) cleanUp() { e.kubeAPIServer.TearDownFn() } -func (e *transformTest) run(unSealSecretFunc unSealSecret, expectedEnvelopePrefix string) { - response, err := e.readRawRecordFromETCD(e.getETCDPath()) +func (e *transformTest) runResource(l kubeapiservertesting.Logger, unSealSecretFunc unSealSecret, expectedEnvelopePrefix, + group, + version, + resource, + name, + namespaceName string, +) { + response, err := e.readRawRecordFromETCD(e.getETCDPathForResource(e.storageConfig.Prefix, group, resource, name, namespaceName)) if err != nil { - e.logger.Errorf("failed to read from etcd: %v", err) + l.Errorf("failed to read from etcd: %v", err) return } if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) { - e.logger.Errorf("expected secret to be prefixed with %s, but got %s", + l.Errorf("expected data to be prefixed with %s, but got %s", expectedEnvelopePrefix, response.Kvs[0].Value) return } // etcd path of the key is used as the authenticated context - need to pass it to decrypt ctx := context.Background() - dataCtx := value.DefaultContext([]byte(e.getETCDPath())) + dataCtx := value.DefaultContext(e.getETCDPathForResource(e.storageConfig.Prefix, group, resource, name, namespaceName)) // Envelope header precedes the cipherTextPayload sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):] transformerConfig, err := e.getEncryptionConfig() if err != nil { - e.logger.Errorf("failed to parse transformer config: %v", err) + l.Errorf("failed to parse transformer config: %v", err) } v, err := unSealSecretFunc(ctx, sealedData, dataCtx, *transformerConfig) if err != nil { - e.logger.Errorf("failed to unseal secret: %v", err) + l.Errorf("failed to unseal secret: %v", err) return } - if !strings.Contains(string(v), secretVal) { - e.logger.Errorf("expected %q after decryption, but got %q", secretVal, string(v)) + if resource == "secrets" { + if !strings.Contains(string(v), secretVal) { + l.Errorf("expected %q after decryption, but got %q", secretVal, string(v)) + } + } else if resource == "configmaps" { + if !strings.Contains(string(v), configMapVal) { + l.Errorf("expected %q after decryption, but got %q", configMapVal, string(v)) + } + } else { + if !strings.Contains(string(v), name) { + l.Errorf("expected %q after decryption, but got %q", name, string(v)) + } } - // Secrets should be un-enveloped on direct reads from Kube API Server. - s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), testSecret, metav1.GetOptions{}) - if err != nil { - e.logger.Errorf("failed to get Secret from %s, err: %v", testNamespace, err) - } - if secretVal != string(s.Data[secretKey]) { - e.logger.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) + // Data should be un-enveloped on direct reads from Kube API Server. + if resource == "secrets" { + s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), testSecret, metav1.GetOptions{}) + if err != nil { + l.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err) + } + if secretVal != string(s.Data[secretKey]) { + l.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) + } + } else if resource == "configmaps" { + s, err := e.restClient.CoreV1().ConfigMaps(namespaceName).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + l.Fatalf("failed to get ConfigMap from %s, err: %v", namespaceName, err) + } + if configMapVal != string(s.Data[configMapKey]) { + l.Errorf("expected %s from KubeAPI, but got %s", configMapVal, string(s.Data[configMapKey])) + } + } else if resource == "pods" { + p, err := e.restClient.CoreV1().Pods(namespaceName).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + l.Fatalf("failed to get Pod from %s, err: %v", namespaceName, err) + } + if p.Name != name { + l.Errorf("expected %s from KubeAPI, but got %s", name, p.Name) + } + } else { + l.Logf("Get object with dynamic client") + fooResource := schema.GroupVersionResource{Group: group, Version: version, Resource: resource} + obj, err := dynamic.NewForConfigOrDie(e.kubeAPIServer.ClientConfig).Resource(fooResource).Namespace(namespaceName).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + l.Fatalf("Failed to get test instance: %v, name: %s", err, name) + } + if obj.GetObjectKind().GroupVersionKind().Group == group && obj.GroupVersionKind().Version == version && obj.GetKind() == resource && obj.GetNamespace() == namespaceName && obj.GetName() != name { + l.Errorf("expected %s from KubeAPI, but got %s", name, obj.GetName()) + } } } @@ -157,12 +208,19 @@ func (e *transformTest) benchmark(b *testing.B) { } } -func (e *transformTest) getETCDPath() string { - return fmt.Sprintf("/%s/secrets/%s/%s", e.storageConfig.Prefix, e.ns.Name, e.secret.Name) +func (e *transformTest) getETCDPathForResource(storagePrefix, group, resource, name, namespaceName string) string { + groupResource := resource + if group != "" { + groupResource = fmt.Sprintf("%s/%s", group, resource) + } + if namespaceName == "" { + return fmt.Sprintf("/%s/%s/%s", storagePrefix, groupResource, name) + } + return fmt.Sprintf("/%s/%s/%s/%s", storagePrefix, groupResource, namespaceName, name) } func (e *transformTest) getRawSecretFromETCD() ([]byte, error) { - secretETCDPath := e.getETCDPath() + secretETCDPath := e.getETCDPathForResource(e.storageConfig.Prefix, "", "secrets", e.secret.Name, e.secret.Namespace) etcdResponse, err := e.readRawRecordFromETCD(secretETCDPath) if err != nil { return nil, fmt.Errorf("failed to read %s from etcd: %v", secretETCDPath, err) @@ -172,7 +230,9 @@ func (e *transformTest) getRawSecretFromETCD() ([]byte, error) { func (e *transformTest) getEncryptionOptions() []string { if e.transformerConfig != "" { - return []string{"--encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName)} + return []string{ + "--encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName), + "--disable-admission-plugins", "ServiceAccount"} } return nil @@ -235,6 +295,60 @@ func (e *transformTest) createSecret(name, namespace string) (*corev1.Secret, er return secret, nil } +func (e *transformTest) createConfigMap(name, namespace string) (*corev1.ConfigMap, error) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string]string{ + configMapKey: configMapVal, + }, + } + if _, err := e.restClient.CoreV1().ConfigMaps(cm.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { + return nil, fmt.Errorf("error while writing configmap: %v", err) + } + + return cm, nil +} + +func gvr(group, version, resource string) schema.GroupVersionResource { + return schema.GroupVersionResource{Group: group, Version: version, Resource: resource} +} + +func createResource(client dynamic.Interface, gvr schema.GroupVersionResource, ns string) (*unstructured.Unstructured, error) { + stubObj, err := getStubObj(gvr) + if err != nil { + return nil, err + } + return client.Resource(gvr).Namespace(ns).Create(context.TODO(), stubObj, metav1.CreateOptions{}) +} + +func getStubObj(gvr schema.GroupVersionResource) (*unstructured.Unstructured, error) { + stub := "" + if data, ok := etcd.GetEtcdStorageDataForNamespace(testNamespace)[gvr]; ok { + stub = data.Stub + } + if len(stub) == 0 { + return nil, fmt.Errorf("no stub data for %#v", gvr) + } + + stubObj := &unstructured.Unstructured{Object: map[string]interface{}{}} + if err := json.Unmarshal([]byte(stub), &stubObj.Object); err != nil { + return nil, fmt.Errorf("error unmarshaling stub for %#v: %v", gvr, err) + } + return stubObj, nil +} + +func (e *transformTest) createPod(namespace string, dynamicInterface dynamic.Interface) (*unstructured.Unstructured, error) { + podGVR := gvr("", "v1", "pods") + pod, err := createResource(dynamicInterface, podGVR, namespace) + if err != nil { + return nil, fmt.Errorf("error while writing pod: %v", err) + } + return pod, nil +} + func (e *transformTest) readRawRecordFromETCD(path string) (*clientv3.GetResponse, error) { rawClient, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport) if err != nil {